{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "<a target=\"_blank\" href=\"https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/main/demos/Main_Demo.ipynb\">\n",
                "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
                "</a>"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Transformer Lens Main Demo Notebook\n",
                "\n",
                "<b style=\"color: red\">To use this notebook, go to Runtime > Change Runtime Type and select GPU as the hardware accelerator.</b>\n",
                "\n",
                "This is a reference notebook covering the main features of the [TransformerLens](https://github.com/TransformerLensOrg/TransformerLens) library for mechanistic interpretability. See [Callum McDougall's tutorial](https://transformerlens-intro.streamlit.app/TransformerLens_&_induction_circuits) for a more structured and gentler introduction to the library"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "**Tips for reading this Colab:**\n",
                "* You can run all this code for yourself! \n",
                "* The graphs are interactive!\n",
                "* Use the table of contents pane in the sidebar to navigate\n",
                "* Collapse irrelevant sections with the dropdown arrows\n",
                "* Search the page using the search in the sidebar, not CTRL+F"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Setup\n",
                "(No need to read)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "DEVELOPMENT_MODE = False\n",
                "# Detect if we're running in Google Colab\n",
                "try:\n",
                "    import google.colab\n",
                "    IN_COLAB = True\n",
                "    print(\"Running as a Colab notebook\")\n",
                "except:\n",
                "    IN_COLAB = False\n",
                "\n",
                "# Install if in Colab\n",
                "if IN_COLAB:\n",
                "    %pip install transformer_lens\n",
                "    %pip install circuitsvis\n",
                "    # Install a faster Node version\n",
                "    !curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash -; sudo apt-get install -y nodejs  # noqa\n",
                "\n",
                "# Hot reload in development mode & not running on the CD\n",
                "if not IN_COLAB:\n",
                "    from IPython import get_ipython\n",
                "    ip = get_ipython()\n",
                "    if not ip.extension_manager.loaded:\n",
                "        ip.extension_manager.load('autoreload')\n",
                "        %autoreload 2\n",
                "        \n",
                "IN_GITHUB = os.getenv(\"GITHUB_ACTIONS\") == \"true\"\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Using renderer: colab\n"
                    ]
                }
            ],
            "source": [
                "# Plotly needs a different renderer for VSCode/Notebooks vs Colab argh\n",
                "import plotly.io as pio\n",
                "if IN_COLAB or not DEVELOPMENT_MODE:\n",
                "    pio.renderers.default = \"colab\"\n",
                "else:\n",
                "    pio.renderers.default = \"notebook_connected\"\n",
                "print(f\"Using renderer: {pio.renderers.default}\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div id=\"circuits-vis-1f2a8687-9cd7\" style=\"margin: 15px 0;\"/>\n",
                            "    <script crossorigin type=\"module\">\n",
                            "    import { render, Hello } from \"https://unpkg.com/circuitsvis@1.43.2/dist/cdn/esm.js\";\n",
                            "    render(\n",
                            "      \"circuits-vis-1f2a8687-9cd7\",\n",
                            "      Hello,\n",
                            "      {\"name\": \"Neel\"}\n",
                            "    )\n",
                            "    </script>"
                        ],
                        "text/plain": [
                            "<circuitsvis.utils.render.RenderedHTML at 0x7f21437f1c30>"
                        ]
                    },
                    "execution_count": 13,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "import circuitsvis as cv\n",
                "# Testing that the library works\n",
                "cv.examples.hello(\"Neel\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Import stuff\n",
                "import torch\n",
                "import torch.nn as nn\n",
                "import einops\n",
                "from fancy_einsum import einsum\n",
                "import tqdm.auto as tqdm\n",
                "import plotly.express as px\n",
                "\n",
                "from jaxtyping import Float\n",
                "from functools import partial"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 15,
            "metadata": {},
            "outputs": [],
            "source": [
                "# import transformer_lens\n",
                "import transformer_lens.utils as utils\n",
                "from transformer_lens.hook_points import (\n",
                "    HookPoint,\n",
                ")  # Hooking utilities\n",
                "from transformer_lens import HookedTransformer, FactoredMatrix"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We turn automatic differentiation off, to save GPU memory, as this notebook focuses on model inference not model training."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 16,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "<torch.autograd.grad_mode.set_grad_enabled at 0x7f213de735e0>"
                        ]
                    },
                    "execution_count": 16,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "torch.set_grad_enabled(False)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Plotting helper functions:"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 298,
            "metadata": {},
            "outputs": [],
            "source": [
                "def imshow(tensor, renderer=None, xaxis=\"\", yaxis=\"\", **kwargs):\n",
                "    px.imshow(utils.to_numpy(tensor), color_continuous_midpoint=0.0, color_continuous_scale=\"RdBu\", labels={\"x\":xaxis, \"y\":yaxis}, **kwargs).show(renderer)\n",
                "\n",
                "def line(tensor, renderer=None, xaxis=\"\", yaxis=\"\", **kwargs):\n",
                "    px.line(utils.to_numpy(tensor), labels={\"x\":xaxis, \"y\":yaxis}, **kwargs).show(renderer)\n",
                "\n",
                "def scatter(x, y, xaxis=\"\", yaxis=\"\", caxis=\"\", renderer=None, **kwargs):\n",
                "    x = utils.to_numpy(x)\n",
                "    y = utils.to_numpy(y)\n",
                "    px.scatter(y=y, x=x, labels={\"x\":xaxis, \"y\":yaxis, \"color\":caxis}, **kwargs).show(renderer)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Introduction"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "This is a demo notebook for [TransformerLens](https://github.com/TransformerLensOrg/TransformerLens), **a library I ([Neel Nanda](https://neelnanda.io)) wrote for doing [mechanistic interpretability](https://distill.pub/2020/circuits/zoom-in/) of GPT-2 Style language models.** The goal of mechanistic interpretability is to take a trained model and reverse engineer the algorithms the model learned during training from its weights. It is a fact about the world today that we have computer programs that can essentially speak English at a human level (GPT-3, PaLM, etc), yet we have no idea how they work nor how to write one ourselves. This offends me greatly, and I would like to solve this! Mechanistic interpretability is a very young and small field, and there are a *lot* of open problems - if you would like to help, please try working on one! **If you want to skill up, check out [my guide to getting started](https://neelnanda.io/getting-started), and if you want to jump into an open problem check out my sequence [200 Concrete Open Problems in Mechanistic Interpretability](https://neelnanda.io/concrete-open-problems).**\n",
                "\n",
                "I wrote this library because after I left the Anthropic interpretability team and started doing independent research, I got extremely frustrated by the state of open source tooling. There's a lot of excellent infrastructure like HuggingFace and DeepSpeed to *use* or *train* models, but very little to dig into their internals and reverse engineer how they work. **This library tries to solve that**, and to make it easy to get into the field even if you don't work at an industry org with real infrastructure! The core features were heavily inspired by [Anthropic's excellent Garcon tool](https://transformer-circuits.pub/2021/garcon/index.html). Credit to Nelson Elhage and Chris Olah for building Garcon and showing me the value of good infrastructure for accelerating exploratory research!\n",
                "\n",
                "The core design principle I've followed is to enable exploratory analysis - one of the most fun parts of mechanistic interpretability compared to normal ML is the extremely short feedback loops! The point of this library is to keep the gap between having an experiment idea and seeing the results as small as possible, to make it easy for **research to feel like play** and to enter a flow state. This notebook demonstrates how the library works and how to use it, but if you want to see how well it works for exploratory research, check out [my notebook analysing Indirect Objection Identification](https://neelnanda.io/exploratory-analysis-demo) or [my recording of myself doing research](https://www.youtube.com/watch?v=yo4QvDn-vsU)!"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Loading and Running Models\n",
                "\n",
                "TransformerLens comes loaded with >40 open source GPT-style models. You can load any of them in with `HookedTransformer.from_pretrained(MODEL_NAME)`. For this demo notebook we'll look at GPT-2 Small, an 80M parameter model, see the Available Models section for info on the rest."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {},
            "outputs": [],
            "source": [
                "device = utils.get_device()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "Using pad_token, but it is not set yet.\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Loaded pretrained model gpt2-small into HookedTransformer\n"
                    ]
                }
            ],
            "source": [
                "# NBVAL_IGNORE_OUTPUT\n",
                "model = HookedTransformer.from_pretrained(\"gpt2-small\", device=device)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "To try the model out, let's find the loss on this text! Models can be run on a single string or a tensor of tokens (shape: [batch, position], all integers), and the possible return types are: \n",
                "* \"logits\" (shape [batch, position, d_vocab], floats), \n",
                "* \"loss\" (the cross-entropy loss when predicting the next token), \n",
                "* \"both\" (a tuple of (logits, loss)) \n",
                "* None (run the model, but don't calculate the logits - this is faster when we only want to use intermediate activations)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 301,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model loss: tensor(4.1758)\n"
                    ]
                }
            ],
            "source": [
                "model_description_text = \"\"\"## Loading Models\n",
                "\n",
                "HookedTransformer comes loaded with >40 open source GPT-style models. You can load any of them in with `HookedTransformer.from_pretrained(MODEL_NAME)`. See my explainer for documentation of all supported models, and this table for hyper-parameters and the name used to load them. Each model is loaded into the consistent HookedTransformer architecture, designed to be clean, consistent and interpretability-friendly. \n",
                "\n",
                "For this demo notebook we'll look at GPT-2 Small, an 80M parameter model. To try the model the model out, let's find the loss on this paragraph!\"\"\"\n",
                "loss = model(model_description_text, return_type=\"loss\")\n",
                "print(\"Model loss:\", loss)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Caching all Activations\n",
                "\n",
                "The first basic operation when doing mechanistic interpretability is to break open the black box of the model and look at all of the internal activations of a model. This can be done with `logits, cache = model.run_with_cache(tokens)`. Let's try this out on the first line of the abstract of the GPT-2 paper.\n",
                "\n",
                "<details><summary>On `remove_batch_dim`</summary>\n",
                "\n",
                "Every activation inside the model begins with a batch dimension. Here, because we only entered a single batch dimension, that dimension is always length 1 and kinda annoying, so passing in the `remove_batch_dim=True` keyword removes it. `gpt2_cache_no_batch_dim = gpt2_cache.remove_batch_dim()` would have achieved the same effect.\n",
                "</details?>"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 302,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "cpu\n"
                    ]
                }
            ],
            "source": [
                "gpt2_text = \"Natural language processing tasks, such as question answering, machine translation, reading comprehension, and summarization, are typically approached with supervised learning on taskspecific datasets.\"\n",
                "gpt2_tokens = model.to_tokens(gpt2_text)\n",
                "print(gpt2_tokens.device)\n",
                "gpt2_logits, gpt2_cache = model.run_with_cache(gpt2_tokens, remove_batch_dim=True)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Let's visualize the attention pattern of all the heads in layer 0, using [Alan Cooney's CircuitsVis library](https://github.com/alan-cooney/CircuitsVis) (based on [Anthropic's PySvelte library](https://github.com/anthropics/PySvelte)). \n",
                "\n",
                "We look this the attention pattern in `gpt2_cache`, an `ActivationCache` object, by entering in the name of the activation, followed by the layer index (here, the activation is called \"attn\" and the layer index is 0). This has shape [head_index, destination_position, source_position], and we use the `model.to_str_tokens` method to convert the text to a list of tokens as strings, since there is an attention weight between each pair of tokens.\n",
                "\n",
                "This visualization is interactive! Try hovering over a token or head, and click to lock. The grid on the top left and for each head is the attention pattern as a destination position by source position grid. It's lower triangular because GPT-2 has **causal attention**, attention can only look backwards, so information can only move forwards in the network.\n",
                "\n",
                "See the ActivationCache section for more on what `gpt2_cache` can do."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 303,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "<class 'transformer_lens.ActivationCache.ActivationCache'>\n",
                        "torch.Size([12, 33, 33])\n"
                    ]
                }
            ],
            "source": [
                "print(type(gpt2_cache))\n",
                "attention_pattern = gpt2_cache[\"pattern\", 0, \"attn\"]\n",
                "print(attention_pattern.shape)\n",
                "gpt2_str_tokens = model.to_str_tokens(gpt2_text)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 304,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Layer 0 Head Attention Patterns:\n"
                    ]
                },
                {
                    "data": {
                        "text/html": [
                            "<div id=\"circuits-vis-44fcf4fd-e695\" style=\"margin: 15px 0;\"/>\n",
                            "    <script crossorigin type=\"module\">\n",
                            "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.0/dist/cdn/esm.js\";\n",
                            "    render(\n",
                            "      \"circuits-vis-44fcf4fd-e695\",\n",
                            "      AttentionPatterns,\n",
                            "      {\"tokens\": [\"<|endoftext|>\", \"Natural\", \" language\", \" processing\", \" tasks\", \",\", \" such\", \" as\", \" question\", \" answering\", \",\", \" machine\", \" translation\", \",\", \" reading\", \" comprehension\", \",\", \" and\", \" summar\", \"ization\", \",\", \" are\", \" typically\", \" approached\", \" with\", \" supervised\", \" learning\", \" on\", \" tasks\", \"pe\", \"cific\", \" datasets\", \".\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9639418125152588, 0.03605816140770912, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8389372825622559, 0.11828784644603729, 0.042774833738803864, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.47436124086380005, 0.13382020592689514, 0.27371731400489807, 0.11810121685266495, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35606446862220764, 0.10184910893440247, 0.2305421680212021, 0.20397399365901947, 0.10757025331258774, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6660143733024597, 0.1686638593673706, 0.045356713235378265, 0.03885499760508537, 0.06775480508804321, 0.013355246745049953, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.38626953959465027, 0.28510934114456177, 0.07609007507562637, 0.059083808213472366, 0.07223353534936905, 0.039796341210603714, 0.08141743391752243, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3775395452976227, 0.18838821351528168, 0.11723987013101578, 0.08685595542192459, 0.06669183075428009, 0.035000164061784744, 0.09692999720573425, 0.031354423612356186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4869752824306488, 0.06781316548585892, 0.07952877879142761, 0.0848078802227974, 0.15902608633041382, 0.02957780286669731, 0.025685928761959076, 0.016474615782499313, 0.05011039599776268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2906550168991089, 0.040134966373443604, 0.14614856243133545, 0.09940595179796219, 0.15389202535152435, 0.039001598954200745, 0.024988984689116478, 0.03184128552675247, 0.10222818702459335, 0.07170350849628448, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39624103903770447, 0.09694176912307739, 0.02727065235376358, 0.02355136163532734, 0.03723451495170593, 0.006502415519207716, 0.08118756115436554, 0.013088458217680454, 0.06990590691566467, 0.24043099582195282, 0.007645336911082268, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24864788353443146, 0.13802054524421692, 0.0923532024025917, 0.08676131069660187, 0.13819700479507446, 0.05914194881916046, 0.032238591462373734, 0.03158241882920265, 0.030489418655633926, 0.03873484954237938, 0.06671835482120514, 0.0371144600212574, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19148437678813934, 0.1617259830236435, 0.07445938885211945, 0.07740951329469681, 0.021961119025945663, 0.033921290189027786, 0.051250241696834564, 0.019519245252013206, 0.03132444992661476, 0.04020152986049652, 0.03874269127845764, 0.21578848361968994, 0.042211681604385376, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.37043172121047974, 0.08681415766477585, 0.024584650993347168, 0.021616313606500626, 0.032388728111982346, 0.005422728601843119, 0.0727522075176239, 0.011272802017629147, 0.06329692155122757, 0.2172681838274002, 0.006367162335664034, 0.029603788629174232, 0.05099846422672272, 0.0071820602752268314, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19737645983695984, 0.04603997990489006, 0.043999817222356796, 0.1337345987558365, 0.054248224943876266, 0.025475721806287766, 0.027563519775867462, 0.021570947021245956, 0.051718246191740036, 0.0645809918642044, 0.028064655140042305, 0.235515758395195, 0.019129818305373192, 0.029963519424200058, 0.021017685532569885, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08907239884138107, 0.01928834617137909, 0.16653534770011902, 0.07281265407800674, 0.04738641157746315, 0.02448790892958641, 0.0289873406291008, 0.01937035843729973, 0.026673004031181335, 0.0731663703918457, 0.025704579427838326, 0.04242357611656189, 0.058694593608379364, 0.028932694345712662, 0.18119071424007416, 0.09527365118265152, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28167298436164856, 0.06441289931535721, 0.01800854131579399, 0.01616961508989334, 0.023183876648545265, 0.003753294236958027, 0.05472243204712868, 0.007909758947789669, 0.04616466909646988, 0.16947270929813385, 0.004361642058938742, 0.02101132832467556, 0.03549071401357651, 0.004932569805532694, 0.09555225074291229, 0.1472632884979248, 0.005917449947446585, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21305488049983978, 0.059123694896698, 0.03382086753845215, 0.02747686579823494, 0.02839355543255806, 0.00842290185391903, 0.04008530080318451, 0.011629253625869751, 0.052951883524656296, 0.1540464460849762, 0.009831812232732773, 0.03610188513994217, 0.04737292230129242, 0.011069183237850666, 0.09972470998764038, 0.13971354067325592, 0.013185345567762852, 0.013994934968650341, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15871700644493103, 0.043878886848688126, 0.08712145686149597, 0.08998468518257141, 0.03073856048285961, 0.034148920327425, 0.0249172393232584, 0.0313919372856617, 0.024823850020766258, 0.019790323451161385, 0.03625482693314552, 0.020694410428404808, 0.04284070432186127, 0.0382089726626873, 0.062346577644348145, 0.10919705033302307, 0.0413760282099247, 0.04916759207844734, 0.05440091714262962, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10485516488552094, 0.12122289091348648, 0.06487478315830231, 0.08768714219331741, 0.03434053063392639, 0.017483938485383987, 0.03415181487798691, 0.015289146453142166, 0.023312106728553772, 0.0283065028488636, 0.018720487132668495, 0.02811194770038128, 0.04190531373023987, 0.0209895521402359, 0.04678507521748543, 0.08659631013870239, 0.023631848394870758, 0.02427317574620247, 0.16702403128147125, 0.010438215918838978, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22901976108551025, 0.05184384807944298, 0.013585175387561321, 0.01233723945915699, 0.018005432561039925, 0.0027703596279025078, 0.04238128289580345, 0.005856249947100878, 0.03614486753940582, 0.13039226830005646, 0.0031534258741885424, 0.015672562643885612, 0.027800437062978745, 0.003554324619472027, 0.07460816949605942, 0.11298278719186783, 0.004272266756743193, 0.006832195445895195, 0.18569743633270264, 0.018073638901114464, 0.005016351584345102, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18869930505752563, 0.03438711538910866, 0.022344743832945824, 0.01997273787856102, 0.016354024410247803, 0.006856544874608517, 0.020859047770500183, 0.005696002393960953, 0.03415917605161667, 0.07260990887880325, 0.00785721093416214, 0.018040234223008156, 0.026904482394456863, 0.009020396508276463, 0.06876449286937714, 0.1757873147726059, 0.010720066726207733, 0.009284532628953457, 0.1925639510154724, 0.02518034167587757, 0.012639064341783524, 0.021299341693520546, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1195831373333931, 0.022259416058659554, 0.03294713795185089, 0.02017025277018547, 0.03565331548452377, 0.013459905050694942, 0.017516477033495903, 0.010057874023914337, 0.025856440886855125, 0.05955959111452103, 0.015084506012499332, 0.015008729882538319, 0.053174663335084915, 0.016597598791122437, 0.04155529662966728, 0.1312934160232544, 0.019296670332551003, 0.015855027362704277, 0.1792508214712143, 0.016183828935027122, 0.02229553833603859, 0.015463395044207573, 0.10187704861164093, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14615534245967865, 0.02672751434147358, 0.01662457175552845, 0.018987704068422318, 0.06278637796640396, 0.015317156910896301, 0.019792228937149048, 0.01422776747494936, 0.02545815333724022, 0.04530354216694832, 0.016364358365535736, 0.037493038922548294, 0.013288671150803566, 0.017496539279818535, 0.03994584083557129, 0.05881756171584129, 0.01926097460091114, 0.024616027250885963, 0.03821970149874687, 0.021577807143330574, 0.02094990387558937, 0.07973218709230423, 0.050176072865724564, 0.17068102955818176, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11321666836738586, 0.036746349185705185, 0.011786574497818947, 0.010274862870573997, 0.020370667800307274, 0.005243880674242973, 0.015918847173452377, 0.005266784690320492, 0.02489170990884304, 0.06593260914087296, 0.005933657754212618, 0.018209027126431465, 0.02102021314203739, 0.006667500827461481, 0.034828800708055496, 0.13742125034332275, 0.007927055470645428, 0.008618656545877457, 0.113771952688694, 0.013557433150708675, 0.009277834556996822, 0.0261213555932045, 0.08499345183372498, 0.1907389909029007, 0.011263738386332989, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13337717950344086, 0.02621661312878132, 0.038271524012088776, 0.07152575254440308, 0.05317767336964607, 0.013925842940807343, 0.007084188517183065, 0.01345015037804842, 0.009841450490057468, 0.01178977731615305, 0.013537583872675896, 0.038154952228069305, 0.04193304106593132, 0.013882283121347427, 0.037071458995342255, 0.13838452100753784, 0.014846336096525192, 0.03156949579715729, 0.05598178505897522, 0.015536688268184662, 0.01595636084675789, 0.04545555263757706, 0.016699660569429398, 0.025325747206807137, 0.036718931049108505, 0.08028542250394821, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10608973354101181, 0.019107729196548462, 0.024468692019581795, 0.027496425434947014, 0.01636582240462303, 0.005011426284909248, 0.010413112118840218, 0.006081154104322195, 0.005301067605614662, 0.011143164709210396, 0.004565389361232519, 0.01896991766989231, 0.004321121145039797, 0.0048149810172617435, 0.029409414157271385, 0.028682038187980652, 0.005097252782434225, 0.0072343493811786175, 0.03412594646215439, 0.010370601899921894, 0.005643271841108799, 0.007283590268343687, 0.029389556497335434, 0.010038797743618488, 0.009134518913924694, 0.5466630458831787, 0.012777918949723244, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10467306524515152, 0.033219389617443085, 0.01534124556928873, 0.00937354750931263, 0.026595445349812508, 0.005787804257124662, 0.01357134710997343, 0.0045548840425908566, 0.028058890253305435, 0.02610722929239273, 0.0063534383662045, 0.013315820135176182, 0.02662825398147106, 0.00688887806609273, 0.06204751506447792, 0.058907054364681244, 0.008068050257861614, 0.007557093631476164, 0.08522786945104599, 0.017075754702091217, 0.009256886318325996, 0.01969575136899948, 0.1261780709028244, 0.13061507046222687, 0.011351034976541996, 0.08984370529651642, 0.04638152942061424, 0.007325293496251106, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0856190174818039, 0.02143894135951996, 0.05641258880496025, 0.057066623121500015, 0.01980220153927803, 0.00672749150544405, 0.005809164140373468, 0.004516264423727989, 0.0031647481955587864, 0.01761520653963089, 0.006174600217491388, 0.08767975121736526, 0.012299864552915096, 0.006350455805659294, 0.017522143200039864, 0.14295215904712677, 0.006585486698895693, 0.007875686511397362, 0.030078930780291557, 0.013907499611377716, 0.007376696448773146, 0.007684788666665554, 0.022160829976201057, 0.012385032139718533, 0.011890065856277943, 0.0866977870464325, 0.19902527332305908, 0.013594484888017178, 0.02958623133599758, 0.0, 0.0, 0.0, 0.0], [0.1406458616256714, 0.013298708014190197, 0.015702951699495316, 0.01735786907374859, 0.022331522777676582, 0.029672272503376007, 0.04172081500291824, 0.018995441496372223, 0.03827711567282677, 0.04863562807440758, 0.03094690851867199, 0.016023889183998108, 0.020880896598100662, 0.0324382521212101, 0.03055812418460846, 0.02280835621058941, 0.03537759184837341, 0.03145160153508186, 0.03497113287448883, 0.01867910847067833, 0.03821909800171852, 0.02257886901497841, 0.06819559633731842, 0.042140960693359375, 0.028620868921279907, 0.037750039249658585, 0.018578054383397102, 0.033768706023693085, 0.036416973918676376, 0.012956847436726093, 0.0, 0.0, 0.0], [0.07168620824813843, 0.06924446672201157, 0.019306888803839684, 0.014161977916955948, 0.016823207959532738, 0.0193806029856205, 0.019257424399256706, 0.0220036618411541, 0.013706528581678867, 0.03578377515077591, 0.018465179949998856, 0.05207167565822601, 0.0200851671397686, 0.019862132146954536, 0.02066211774945259, 0.04725164175033569, 0.021076709032058716, 0.03678721562027931, 0.024324074387550354, 0.0038275495171546936, 0.023920681327581406, 0.00853322446346283, 0.026241611689329147, 0.027380069717764854, 0.034612007439136505, 0.02288416028022766, 0.10047900676727295, 0.06913495808839798, 0.02547456882894039, 0.06495602428913116, 0.03061540424823761, 0.0, 0.0], [0.0691077783703804, 0.0370122529566288, 0.038621146231889725, 0.059333253651857376, 0.015923552215099335, 0.007918557152152061, 0.010371049866080284, 0.006615668535232544, 0.002520076697692275, 0.026019373908638954, 0.007905226200819016, 0.029652021825313568, 0.04000623896718025, 0.008451340720057487, 0.010741155594587326, 0.050275612622499466, 0.009428860619664192, 0.013601025566458702, 0.05036922171711922, 0.03176714479923248, 0.010793023742735386, 0.007216802332550287, 0.006478433031588793, 0.014800610020756721, 0.021585965529084206, 0.15769486129283905, 0.0888475850224495, 0.01901692897081375, 0.029729334637522697, 0.03316137567162514, 0.05088365823030472, 0.034150853753089905, 0.0], [0.14375515282154083, 0.016811039298772812, 0.009386657737195492, 0.006830316502600908, 0.011656849645078182, 0.0015672278823330998, 0.019711483269929886, 0.0023980415426194668, 0.021235883235931396, 0.046836815774440765, 0.001690515549853444, 0.005827185697853565, 0.011979996226727962, 0.0018251386936753988, 0.042313314974308014, 0.05491373687982559, 0.0021786566358059645, 0.0024170074611902237, 0.096040740609169, 0.005752879660576582, 0.0025577901396900415, 0.0071212295442819595, 0.08889932930469513, 0.10852077603340149, 0.00517905130982399, 0.036577362567186356, 0.024719929322600365, 0.00373476417735219, 0.031077083200216293, 0.016887929290533066, 0.094508595764637, 0.07171522825956345, 0.0033723199740052223]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00042466612649150193, 0.9995753169059753, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005621905438601971, 0.016407258808612823, 0.9830306172370911, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011627542553469539, 0.02168194390833378, 0.003762035397812724, 0.9733933210372925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.7244404666125774e-05, 0.00017202268645633012, 0.0002814393083099276, 0.002742142416536808, 0.9967671632766724, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008268453180789948, 0.00023985578445717692, 7.361903408309445e-05, 6.437727279262617e-05, 0.0001756635756464675, 0.9911779761314392, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012215040624141693, 0.00540044903755188, 0.0016716319369152188, 0.00040775613160803914, 0.0006163661018945277, 0.0010931181022897363, 0.9895892143249512, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012459801509976387, 0.0009121242328546941, 0.0005976730608381331, 0.00013656924420502037, 0.00033041174174286425, 0.0015722811222076416, 0.0038808262906968594, 0.9913241863250732, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00028217461658641696, 0.004068182315677404, 0.0026605145540088415, 0.0013093099696561694, 0.008030476048588753, 0.00028790938085876405, 0.00022922919015400112, 0.00039484212175011635, 0.9827372431755066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [3.473927063168958e-05, 0.000395383860450238, 0.00013272698561195284, 0.00025852289400063455, 0.0010855571599677205, 9.198045154334977e-05, 0.0003267078718636185, 0.0005427449941635132, 0.006105928216129541, 0.9910257458686829, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0033785109408199787, 5.090872218715958e-05, 1.6452064301120117e-05, 1.6926183889154345e-05, 4.181411713943817e-05, 0.4939400851726532, 0.0001298127754125744, 0.0008837342611514032, 3.221194492653012e-05, 2.725212470977567e-05, 0.5014824271202087, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.416040509473532e-05, 0.0013417209265753627, 0.001261358498595655, 0.0021450743079185486, 0.004042362328618765, 0.0004830524558201432, 0.00011582644219743088, 0.000152035805513151, 2.692525231395848e-05, 0.000126751241623424, 0.00031289877369999886, 0.9899077415466309, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00037822299054823816, 0.0009837561519816518, 0.03934125974774361, 0.002732248045504093, 0.0036680190823972225, 0.00011039189848816022, 0.0001293104432988912, 0.0002174347755499184, 0.00010623304115142673, 0.0007748189964331686, 6.647665577474982e-05, 0.0003148667747154832, 0.9511768817901611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021017210092395544, 2.443683843011968e-05, 7.788769835315179e-06, 8.651618372823577e-06, 2.0140032575000077e-05, 0.2997134029865265, 7.52526757423766e-05, 0.0004898310289718211, 1.8459444618201815e-05, 1.534457078378182e-05, 0.32833901047706604, 4.1757990402402356e-05, 6.4691921579651535e-06, 0.369137704372406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00014968300820328295, 0.0001129658630816266, 0.00036294819437898695, 0.0001859118783613667, 0.00016460861661471426, 4.143221667618491e-05, 2.8764718081220053e-05, 7.786935748299584e-05, 0.0009200966451317072, 0.010340160690248013, 2.7572314138524234e-05, 1.783318839443382e-05, 0.00033054433879442513, 2.4375658540520817e-05, 0.9872152805328369, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010753806418506429, 0.0021781891118735075, 0.002042608568444848, 0.004251919221132994, 0.00698988838121295, 2.5118722987826914e-05, 0.0007779028965160251, 0.0005783533561043441, 0.0029378405306488276, 0.03322531282901764, 1.719921601761598e-05, 0.0008936477825045586, 0.0015238532796502113, 1.4656765415566042e-05, 0.006222627125680447, 0.9382133483886719, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013634879142045975, 1.3226032024249434e-05, 4.013933903479483e-06, 4.80306016470422e-06, 1.0257448593620211e-05, 0.1966594010591507, 4.527257624431513e-05, 0.0002776258625090122, 1.1714419088093564e-05, 9.473317732044961e-06, 0.22919629514217377, 2.643059087859001e-05, 4.101846570847556e-06, 0.26576104760169983, 8.515345143678132e-06, 4.536019787337864e-06, 0.3065996766090393, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008924759458750486, 0.00013490796845871955, 4.779840674018487e-05, 5.803714520880021e-05, 0.00010480164200998843, 0.01279909536242485, 0.00071682472480461, 0.03257950767874718, 2.6449932192917913e-05, 0.00011185074981767684, 0.01188427209854126, 4.010266638942994e-05, 5.555409370572306e-05, 0.01237786840647459, 0.00010783470497699454, 5.404360490501858e-05, 0.013122131116688251, 0.9148864150047302, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [5.075319677416701e-06, 4.396143413032405e-05, 3.3985117624979466e-05, 7.940302020870149e-05, 5.4779120546299964e-05, 7.921551627987355e-07, 9.313333976024296e-06, 7.727086995146237e-06, 8.597279520472512e-05, 0.00012274044274818152, 5.141489509696839e-07, 1.7027937246893998e-06, 3.8341793697327375e-05, 4.450975836789439e-07, 0.000139282361487858, 0.0003275807248428464, 3.9948051266947004e-07, 3.948138328269124e-06, 0.9990440011024475, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [9.346189472125843e-05, 0.001839571981690824, 0.002523313742130995, 0.018087247386574745, 0.0029363788198679686, 0.0002733580768108368, 4.872979116044007e-05, 0.00042127762571908534, 0.00015624362276867032, 0.0009748342563398182, 0.0002053358475677669, 0.0010228834580630064, 0.0019548034761101007, 0.00019470365077722818, 0.0011294210562482476, 0.0016656102379783988, 0.000187340410775505, 0.0009503443725407124, 0.00044551832252182066, 0.9648895859718323, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010859724134206772, 8.512319254805334e-06, 2.5307267605967354e-06, 3.062518317165086e-06, 5.97512462263694e-06, 0.13292613625526428, 3.3453237847425044e-05, 0.00018891259969677776, 8.477749361190945e-06, 6.540428785228869e-06, 0.16445933282375336, 1.8130023818230256e-05, 3.0627520573034417e-06, 0.19719170033931732, 6.429842869692948e-06, 3.4447368761902908e-06, 0.23317135870456696, 0.0022796241100877523, 3.7134373087610584e-06, 3.528806701069698e-05, 0.2685583829879761, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000419795309426263, 0.00011805015674326569, 0.00014240489690564573, 3.7967849493725225e-05, 0.00019043164502363652, 0.0017651219386607409, 0.0005709825782105327, 0.0005008854204788804, 8.840746886562556e-05, 0.00014208679203875363, 0.0016639836831018329, 3.348105383338407e-05, 2.4414008294115774e-05, 0.0017546507297083735, 6.520312308566645e-05, 2.41426496359054e-05, 0.0018299142830073833, 0.0015691012376919389, 3.974877472501248e-05, 0.00015712402819190174, 0.0018554450944066048, 0.987006664276123, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.501784421037883e-05, 0.001437984756194055, 6.345157453324646e-05, 0.00010864849900826812, 0.00015633212751708925, 3.210126124031376e-06, 0.0022032808046787977, 0.0002207659708801657, 5.2403021982172504e-05, 4.88158839289099e-05, 2.264461500089965e-06, 1.5327248547691852e-05, 4.157144758210052e-06, 2.0228428638802143e-06, 6.2968028942123055e-06, 4.8486737796338275e-05, 1.992900934055797e-06, 3.247011045459658e-05, 0.0012695115292444825, 1.963266549864784e-05, 1.8090950106852688e-06, 0.0005810299189761281, 0.9936450719833374, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.450158591382205e-05, 0.000613938900642097, 0.0009361205156892538, 0.0008487807353958488, 0.0028506340458989143, 1.0365061825723387e-05, 0.00021614297293126583, 0.0001739754807204008, 0.0020508412271738052, 0.005805302411317825, 8.055246325966436e-06, 8.086599700618535e-05, 0.000770243932493031, 7.2881916821643244e-06, 0.0010576170170679688, 0.0022755973041057587, 6.663267868134426e-06, 0.00011621040175668895, 0.0005972454091534019, 8.736305608181283e-05, 6.332331849989714e-06, 6.0964586737100035e-05, 6.0905833379365504e-05, 0.9812840819358826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.000584874302148819, 0.0001590918836882338, 1.0836471119546331e-05, 7.36547663109377e-05, 0.00011349536362104118, 0.0008256656583398581, 0.00031911046244204044, 0.018529996275901794, 1.0226590347883757e-05, 4.958722638548352e-05, 0.0007716426043771207, 4.454815643839538e-05, 9.865044376056176e-06, 0.0008067175513133407, 2.2673888452118263e-05, 1.2464140127121937e-05, 0.000844910042360425, 0.008790099062025547, 3.579241456463933e-05, 3.662862945930101e-05, 0.0008917520754039288, 0.0010791769018396735, 0.000370846304576844, 0.00010837137961061671, 0.9654980301856995, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6459925973322242e-05, 0.00017152422515209764, 3.208317866665311e-05, 0.00010234172805212438, 0.0026318784803152084, 9.886175575957168e-06, 3.250852023484185e-05, 3.7417539715534076e-05, 0.00012631528079509735, 4.9912112444872037e-05, 8.302540663862601e-06, 8.443406113656238e-05, 3.127968739136122e-05, 7.633363566128537e-06, 1.0101362931891344e-05, 5.667375444318168e-05, 7.442129117407603e-06, 2.7689680791809224e-05, 1.8418375475448556e-05, 2.879437261071871e-06, 6.840427886345424e-06, 4.279875156498747e-06, 0.00043176571489311755, 0.0001761750172590837, 8.995608368422836e-05, 0.9958257675170898, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [2.6368044927949086e-05, 0.00018008689221460372, 0.00018082931637763977, 0.0003046471974812448, 0.00039390864549204707, 4.674839510698803e-05, 2.7231690182816237e-05, 4.8734902520664036e-05, 0.00029131610062904656, 0.0004206164157949388, 3.8046357076382264e-05, 0.0002524506417103112, 5.6067321565933526e-05, 3.8203645090106875e-05, 0.0015365113504230976, 0.0012537208385765553, 3.5934212064603344e-05, 2.3036614948068745e-05, 0.0001803624036256224, 0.00012266075646039099, 3.5177803511032835e-05, 6.924677290953696e-05, 0.00011267293302807957, 0.000850738026201725, 0.00014361874491441995, 0.00023527955636382103, 0.9930956959724426, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0007048251572996378, 5.675383727066219e-05, 3.351286068209447e-06, 7.472657671314664e-06, 1.785946005838923e-05, 0.0008156524272635579, 4.952957897330634e-05, 0.0013530978467315435, 3.741756154340692e-05, 0.00014378006744664162, 0.0007639339892193675, 9.991676051868126e-06, 2.884435843952815e-06, 0.0007552225724793971, 0.00010159592784475535, 3.1205381674226373e-06, 0.000806094438303262, 0.0011406750418245792, 1.4343104339786805e-05, 9.910167136695236e-06, 0.0008649182855151594, 8.663265907671303e-05, 3.4695403883233666e-05, 0.00010265811579301953, 0.007663467433303595, 4.1878953197738156e-05, 3.4908664474642137e-06, 0.9844048619270325, 0.0, 0.0, 0.0, 0.0, 0.0], [2.0288314772187732e-05, 2.954267074528616e-05, 5.0374983402434736e-05, 0.00097783247474581, 0.3728252649307251, 6.6613069975574035e-06, 1.573364352225326e-05, 3.981243571615778e-05, 0.0002235322172055021, 0.000126743339933455, 5.118435637996299e-06, 0.00024116155691444874, 1.2973826414963696e-05, 4.800691385753453e-06, 2.332163967366796e-05, 7.723519956925884e-05, 4.5934334593766835e-06, 1.9647124645416625e-05, 0.00021129944070708007, 1.1453501429059543e-05, 4.380481641419465e-06, 1.4442671272263397e-05, 3.676746564451605e-05, 0.000118453215691261, 3.797715908149257e-05, 0.0007802762556821108, 0.0004048367263749242, 1.042955591401551e-05, 0.6236649751663208, 0.0, 0.0, 0.0, 0.0], [2.058288737316616e-05, 8.831218292471021e-05, 0.00020454842888284475, 0.0003018889983650297, 8.223871554946527e-05, 1.970437733689323e-05, 0.0001408507232554257, 2.8963118893443607e-05, 7.669363185414113e-06, 3.7246059946483e-05, 1.673675433266908e-05, 6.404684245353565e-05, 0.0006910567753948271, 1.6027554011088796e-05, 0.00015603665087837726, 0.00014825885591562837, 1.5700054063927382e-05, 9.155381849268451e-05, 8.525057637598366e-05, 4.904503384750569e-06, 1.5784382412675768e-05, 5.293102003633976e-05, 0.0005298344185575843, 0.0005658378941006958, 6.167311948956922e-05, 6.729478627676144e-05, 0.000307743699522689, 1.0369139090471435e-05, 6.79933000355959e-05, 0.9960988759994507, 0.0, 0.0, 0.0], [1.3371342902246397e-05, 0.000982154393568635, 0.0004154824127908796, 0.0001144234774983488, 0.00038730859523639083, 5.660870101564797e-06, 0.0012746285647153854, 0.0005708064418286085, 0.0006383642321452498, 0.0005776673788204789, 4.127737156522926e-06, 9.161751222563908e-06, 9.142841008724645e-05, 3.7748709473817144e-06, 1.357534529233817e-05, 0.00029161793645471334, 3.4474919630156364e-06, 7.899192132754251e-05, 0.003189126728102565, 4.885087946604472e-06, 3.1652864436182426e-06, 1.4087467206991278e-05, 0.00015671442088205367, 0.00035448954440653324, 0.0001726548798615113, 0.0013050639536231756, 0.00021867685427423567, 2.6776768208947033e-05, 0.0002646023058332503, 1.333455293206498e-05, 0.9888004064559937, 0.0, 0.0], [2.745508936641272e-05, 0.00016438246530015022, 7.99642366473563e-05, 0.001191496616229415, 0.0007883626385591924, 2.658417088241549e-06, 3.0057593903620727e-05, 7.457976153091295e-06, 0.0001494098105467856, 2.8857160941697657e-05, 1.873827500276093e-06, 0.00033288035774603486, 5.16086038260255e-05, 1.757749942044029e-06, 0.00012656577746383846, 0.000142670251079835, 1.6954454622464254e-06, 2.1952595488983206e-05, 0.00023040804080665112, 4.429338878253475e-05, 1.6103307416415191e-06, 2.7008050892618485e-05, 0.0002388486755080521, 0.0001904603559523821, 9.496146049059462e-06, 0.0004466440004762262, 0.00022095769236329943, 5.379181402531685e-06, 0.0006956413271836936, 0.0001547078281873837, 0.0002548546181060374, 0.9943286776542664, 0.0], [0.0062310462817549706, 9.722806862555444e-05, 6.871583536849357e-06, 2.1151136024855077e-05, 5.828053326695226e-05, 0.007238902151584625, 2.0987936295568943e-05, 0.00025459096650592983, 6.243847747100517e-05, 2.092468275805004e-05, 0.007872143760323524, 5.5853193771326914e-05, 9.868776942312252e-06, 0.009169402532279491, 7.203003042377532e-05, 7.068687409628183e-06, 0.010345960967242718, 0.0013096530456095934, 3.803677100222558e-05, 8.022689144127071e-05, 0.012053250335156918, 4.071994771948084e-05, 3.6860715226794127e-06, 3.4713455534074455e-05, 0.0005061827832832932, 8.918941603042185e-05, 2.9112034098943695e-05, 0.0012772803893312812, 8.489640458719805e-05, 0.0001844740763772279, 0.00013425741053652018, 6.813048094045371e-05, 0.9425214529037476]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.943029522895813, 0.056970469653606415, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9185556173324585, 0.03280005604028702, 0.048644375056028366, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8779287934303284, 0.056434307247400284, 0.04271191731095314, 0.02292500250041485, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.804131805896759, 0.02909819595515728, 0.07556723803281784, 0.05643589794635773, 0.034766778349876404, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4943104088306427, 0.020183557644486427, 0.027966545894742012, 0.01831907406449318, 0.0314420647919178, 0.407778263092041, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6057476997375488, 0.02924242988228798, 0.09491512924432755, 0.07609355449676514, 0.06614666432142258, 0.08705785125494003, 0.040796659886837006, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44838201999664307, 0.04542431980371475, 0.0740148276090622, 0.06864843517541885, 0.09376634657382965, 0.0877426490187645, 0.06534271687269211, 0.11667868494987488, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.49169260263442993, 0.137820765376091, 0.03955020010471344, 0.06153317168354988, 0.04539967328310013, 0.04073144495487213, 0.06228702515363693, 0.05861866846680641, 0.06236643344163895, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5404126048088074, 0.04442667216062546, 0.03957851976156235, 0.041888073086738586, 0.07529859244823456, 0.0466950386762619, 0.0484752431511879, 0.05500517413020134, 0.08293059468269348, 0.025289515033364296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2827354967594147, 0.01423418428748846, 0.01764773577451706, 0.011433068662881851, 0.02174171432852745, 0.266653835773468, 0.015403532423079014, 0.04734927415847778, 0.017767537385225296, 0.01392605435103178, 0.2911074161529541, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3449663519859314, 0.04611678421497345, 0.05771546810865402, 0.11131863296031952, 0.11289364099502563, 0.027930330485105515, 0.038591884076595306, 0.05656527727842331, 0.05864056944847107, 0.06648591160774231, 0.02611437998712063, 0.05266086757183075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46992170810699463, 0.03201567754149437, 0.1077289879322052, 0.027006778866052628, 0.04465882480144501, 0.022773556411266327, 0.023117080330848694, 0.025491518899798393, 0.04950270429253578, 0.026573920622467995, 0.01970885880291462, 0.06337959319353104, 0.08812078088521957, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20661766827106476, 0.01037930604070425, 0.012261925265192986, 0.008319089189171791, 0.01600709557533264, 0.1952773481607437, 0.0114532969892025, 0.03475672006607056, 0.013073587790131569, 0.010938971303403378, 0.21602490544319153, 0.005866494029760361, 0.02364230528473854, 0.23538126051425934, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3399347960948944, 0.029161639511585236, 0.09540707617998123, 0.03395191207528114, 0.08440457284450531, 0.012559536844491959, 0.02935865893959999, 0.024564161896705627, 0.10622433573007584, 0.04689215123653412, 0.011469588615000248, 0.006369189824908972, 0.11145280301570892, 0.011317988857626915, 0.05693149194121361, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4040881395339966, 0.02419566549360752, 0.038910042494535446, 0.014727436006069183, 0.024456584826111794, 0.038450006395578384, 0.03923037648200989, 0.0371713824570179, 0.06030004471540451, 0.041985440999269485, 0.03716716542840004, 0.016391228884458542, 0.039289675652980804, 0.03772977739572525, 0.13448573648929596, 0.011421291157603264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15876434743404388, 0.008203915320336819, 0.009295256808400154, 0.0062217144295573235, 0.011793207377195358, 0.15069787204265594, 0.00885195191949606, 0.0263135377317667, 0.010186920873820782, 0.008433720096945763, 0.16762235760688782, 0.004420032724738121, 0.018211791291832924, 0.18350805342197418, 0.020927326753735542, 0.006447788793593645, 0.2001001089811325, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1753920316696167, 0.020569661632180214, 0.018291428685188293, 0.009298022836446762, 0.01737789623439312, 0.04253473877906799, 0.020701589062809944, 0.05044392868876457, 0.02543804980814457, 0.01721823401749134, 0.04311532527208328, 0.013349822722375393, 0.028528599068522453, 0.04597253352403641, 0.03408820927143097, 0.019834185019135475, 0.04992839694023132, 0.3679172694683075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26033341884613037, 0.017148377373814583, 0.03745277598500252, 0.07594797760248184, 0.04674699902534485, 0.018068524077534676, 0.031346458941698074, 0.03741515055298805, 0.07175808399915695, 0.05872533842921257, 0.0170787051320076, 0.040305934846401215, 0.05706356093287468, 0.01711358316242695, 0.10491344332695007, 0.04670536518096924, 0.017230207100510597, 0.02468213252723217, 0.01996397227048874, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.30547067523002625, 0.05190497264266014, 0.043468981981277466, 0.021846864372491837, 0.021017208695411682, 0.03390473127365112, 0.041904695332050323, 0.03909289836883545, 0.02887197583913803, 0.023003440350294113, 0.032057985663414, 0.02333451434969902, 0.07110592722892761, 0.032900869846343994, 0.061641909182071686, 0.03183262422680855, 0.033767640590667725, 0.04571491852402687, 0.035015229135751724, 0.022141896188259125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12242944538593292, 0.006105302833020687, 0.006670293863862753, 0.004581776913255453, 0.009337793104350567, 0.1138179749250412, 0.006783355493098497, 0.019719908013939857, 0.007580860517919064, 0.006613647099584341, 0.12765929102897644, 0.003502656938508153, 0.014002319425344467, 0.14000986516475677, 0.015684882178902626, 0.005092609208077192, 0.1533622145652771, 0.03527413681149483, 0.022465627640485764, 0.006954461336135864, 0.17235170304775238, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20538480579853058, 0.034778494387865067, 0.014682917855679989, 0.031183309853076935, 0.030931269749999046, 0.021952776238322258, 0.032908640801906586, 0.05740533769130707, 0.05587517470121384, 0.04864276573061943, 0.023520752787590027, 0.015108545310795307, 0.027386309579014778, 0.02451845072209835, 0.06060453876852989, 0.034776173532009125, 0.026137826964259148, 0.051684342324733734, 0.06281407177448273, 0.020291481167078018, 0.028601743280887604, 0.09081023931503296, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23978610336780548, 0.03132349252700806, 0.05037751421332359, 0.01586943119764328, 0.039014630019664764, 0.022805538028478622, 0.042853571474552155, 0.02888229675590992, 0.040462613105773926, 0.03410725295543671, 0.0226445309817791, 0.03923073410987854, 0.07238569110631943, 0.02234550192952156, 0.04958105459809303, 0.03193334862589836, 0.023325597867369652, 0.04521358758211136, 0.03055434115231037, 0.02287650667130947, 0.024732287973165512, 0.05549480766057968, 0.014199561439454556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2358168512582779, 0.02055830880999565, 0.04375007376074791, 0.029704876244068146, 0.03703880310058594, 0.01495345402508974, 0.040043096989393234, 0.027184369042515755, 0.0457618348300457, 0.03809259459376335, 0.014181870967149734, 0.037891559302806854, 0.06518241763114929, 0.014182929880917072, 0.054894935339689255, 0.02372094802558422, 0.014592778868973255, 0.025570014491677284, 0.07356180995702744, 0.03918234258890152, 0.014925528317689896, 0.046288665384054184, 0.027801062911748886, 0.01511886715888977, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14254963397979736, 0.012010159902274609, 0.01688128523528576, 0.020742565393447876, 0.03245178237557411, 0.029626017436385155, 0.03029521182179451, 0.05620871111750603, 0.029608752578496933, 0.02948184497654438, 0.03263987973332405, 0.010038236156105995, 0.040786173194646835, 0.034627851098775864, 0.03391636162996292, 0.02015574276447296, 0.0368431992828846, 0.060646891593933105, 0.04744710400700569, 0.03252531215548515, 0.04040037468075752, 0.059477631002664566, 0.031293999403715134, 0.04792548343539238, 0.07141982764005661, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22229072451591492, 0.0591113418340683, 0.037026748061180115, 0.040590159595012665, 0.027254492044448853, 0.01917477324604988, 0.03171536326408386, 0.020462142303586006, 0.038113292306661606, 0.01992788352072239, 0.01853892207145691, 0.015436715446412563, 0.04536492004990578, 0.0193557720631361, 0.050358306616544724, 0.03328137844800949, 0.02017977274954319, 0.03679434582591057, 0.04331386461853981, 0.028476400300860405, 0.02131732553243637, 0.04771288111805916, 0.01310704369097948, 0.02633604407310486, 0.030211374163627625, 0.03454802557826042, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28438133001327515, 0.023870108649134636, 0.04641878977417946, 0.010260095819830894, 0.033909872174263, 0.018301473930478096, 0.0237088855355978, 0.02476685680449009, 0.02576776035130024, 0.022968821227550507, 0.016735462471842766, 0.013406937941908836, 0.045986607670784, 0.01667323336005211, 0.0810638889670372, 0.05033262446522713, 0.01672617346048355, 0.01990482211112976, 0.030325349420309067, 0.010146327316761017, 0.017318250611424446, 0.01904098130762577, 0.011081117205321789, 0.052046410739421844, 0.0333535373210907, 0.03880884498357773, 0.012695522047579288, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11854353547096252, 0.013350957073271275, 0.013422027230262756, 0.030273228883743286, 0.02616296149790287, 0.02077619917690754, 0.0212801992893219, 0.03798936307430267, 0.03536442294716835, 0.036651793867349625, 0.0220040250569582, 0.015380030497908592, 0.030023114755749702, 0.023299677297472954, 0.030007556080818176, 0.014861371368169785, 0.024521954357624054, 0.03999800980091095, 0.039001550525426865, 0.03600768372416496, 0.026851210743188858, 0.06795874983072281, 0.038931287825107574, 0.057258058339357376, 0.06659835577011108, 0.03075684979557991, 0.023343218490481377, 0.05938265845179558, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2509184181690216, 0.011351888999342918, 0.023033956065773964, 0.018849771469831467, 0.013208831660449505, 0.016430404037237167, 0.03763698786497116, 0.02151942066848278, 0.03823148459196091, 0.031223095953464508, 0.016081098467111588, 0.01717902161180973, 0.08239644765853882, 0.015775399282574654, 0.049693118780851364, 0.03397900611162186, 0.01646914891898632, 0.025656120851635933, 0.05326932296156883, 0.024380972608923912, 0.017032910138368607, 0.031062565743923187, 0.014834271743893623, 0.043107807636260986, 0.027818365022540092, 0.015127955004572868, 0.012749838642776012, 0.026852698996663094, 0.014129591174423695, 0.0, 0.0, 0.0, 0.0], [0.19256943464279175, 0.02283368818461895, 0.01449588593095541, 0.02805502340197563, 0.032904304563999176, 0.018577098846435547, 0.023776713758707047, 0.014988910406827927, 0.027755489572882652, 0.019952144473791122, 0.018426893278956413, 0.026808498427271843, 0.040261950343847275, 0.018957339227199554, 0.01998024992644787, 0.039050325751304626, 0.01949434168636799, 0.03071432374417782, 0.07932323962450027, 0.03619767725467682, 0.020379580557346344, 0.023319309577345848, 0.018723854795098305, 0.05692768096923828, 0.02392754890024662, 0.03924325481057167, 0.021783528849482536, 0.020374208688735962, 0.04178832098841667, 0.008409242145717144, 0.0, 0.0, 0.0], [0.1891939789056778, 0.014352066442370415, 0.02782934345304966, 0.01893605850636959, 0.054552361369132996, 0.02430238202214241, 0.020752564072608948, 0.03050179034471512, 0.01690042018890381, 0.02990402840077877, 0.023111781105399132, 0.021664857864379883, 0.03335981443524361, 0.023050500079989433, 0.027927029877901077, 0.026253480464220047, 0.0242579597979784, 0.02439497597515583, 0.017666665837168694, 0.02208109013736248, 0.025441501289606094, 0.023996712639927864, 0.015941403806209564, 0.021863479167222977, 0.04876275360584259, 0.010234066285192966, 0.02576298452913761, 0.06192326545715332, 0.06737111508846283, 0.018184112384915352, 0.009525517001748085, 0.0, 0.0], [0.2711195945739746, 0.05460575222969055, 0.03970598056912422, 0.03512897342443466, 0.020313773304224014, 0.008183859288692474, 0.022556623443961143, 0.01403643749654293, 0.026198463514447212, 0.03235534578561783, 0.007185029797255993, 0.010813496075570583, 0.051626089960336685, 0.007057538256049156, 0.04232775792479515, 0.018505984917283058, 0.006952998228371143, 0.012550835497677326, 0.037855181843042374, 0.014787385240197182, 0.007130769081413746, 0.01895672269165516, 0.010758965276181698, 0.023540517315268517, 0.016256190836429596, 0.007915409281849861, 0.03652311861515045, 0.01456227246671915, 0.01925145648419857, 0.010911806486546993, 0.034396685659885406, 0.06592899560928345, 0.0], [0.14836545288562775, 0.014506906270980835, 0.007646297570317984, 0.011038756929337978, 0.026302505284547806, 0.013194817118346691, 0.027184877544641495, 0.01739220693707466, 0.0161221232265234, 0.03288078308105469, 0.014050274156033993, 0.007167344447225332, 0.016120057553052902, 0.014647016301751137, 0.018450886011123657, 0.013102116994559765, 0.015283223241567612, 0.014542976394295692, 0.05961216986179352, 0.022383902221918106, 0.016796978190541267, 0.10928261280059814, 0.07263999432325363, 0.08393707126379013, 0.025248995050787926, 0.03774847462773323, 0.016226934269070625, 0.024953853338956833, 0.04216582328081131, 0.0045258100144565105, 0.014491367153823376, 0.019145715981721878, 0.022841639816761017]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09646999090909958, 0.903529942035675, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04325234889984131, 0.0817774161696434, 0.8749701976776123, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09995415061712265, 0.025312775745987892, 0.02010808140039444, 0.8546250462532043, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024889405816793442, 0.003207360627129674, 0.0018421609420329332, 0.022361507639288902, 0.9476995468139648, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10732369869947433, 0.017841659486293793, 0.019553322345018387, 0.04333317279815674, 0.10211493074893951, 0.7098332047462463, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006426365114748478, 0.00044797908049076796, 0.00014756617019884288, 0.00046936768922023475, 0.0014411915326490998, 0.0038597022648900747, 0.98720782995224, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010354578262194991, 0.00019901855557691306, 0.00016020408656913787, 6.937277794349939e-05, 0.000386741739930585, 0.005171590019017458, 0.8964056372642517, 0.0965719148516655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0012883319286629558, 0.00032334530260413885, 0.00026527183945290744, 0.0002549061318859458, 0.00020129804033786058, 0.000100490047771018, 0.0005700875190086663, 0.0004091293376404792, 0.9965870380401611, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00018741549865808338, 1.6196529031731188e-05, 4.228103534842376e-06, 0.00028758824919350445, 1.112571499106707e-05, 9.805654372030403e-06, 0.00015567190712317824, 7.632956840097904e-05, 0.0034869106020778418, 0.9957646131515503, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015741512179374695, 0.0006393603980541229, 0.0004571523459162563, 0.0009912523673847318, 0.002114036353304982, 0.0180897768586874, 0.0471203587949276, 0.07010914385318756, 0.061528280377388, 0.27690389752388, 0.5063051581382751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006166594685055315, 0.0005207074573263526, 4.6115106670185924e-05, 0.001461300067603588, 0.0005623751203529537, 4.447599349077791e-05, 0.0003653964085970074, 0.0002860165841411799, 0.004506128840148449, 0.005816522985696793, 0.0007244535954669118, 0.9850497841835022, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010478552430868149, 3.062811811105348e-05, 0.00017344979278277606, 5.423108450486325e-05, 6.388417386915535e-05, 1.1261126928729936e-05, 1.716897531878203e-05, 1.3931280591350514e-05, 0.002076038159430027, 0.000269268115516752, 0.00015268487913999707, 0.003684425260871649, 0.992405116558075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010746270418167114, 0.00025354328681714833, 0.00015322922263294458, 0.0002928703906945884, 0.0005376797635108232, 0.004423548001796007, 0.009884343482553959, 0.012843053787946701, 0.012738604098558426, 0.05966852977871895, 0.10773004591464996, 0.024745715782046318, 0.07808706164360046, 0.6778955459594727, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00041217965190298855, 2.6218716811854392e-05, 1.775590681063477e-05, 0.00019181867537554353, 3.2979528441501316e-06, 3.912675310857594e-06, 1.0438079698360525e-05, 4.906840786134126e-06, 0.0005868562730029225, 0.0030381486285477877, 3.693114558700472e-05, 0.000772429455537349, 0.009622587822377682, 0.00016094365855678916, 0.9851115345954895, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00010886572999879718, 3.584545311241527e-06, 5.191652689973125e-06, 3.083834599237889e-05, 1.8408729374641553e-05, 7.954758416417462e-07, 3.3532076031406177e-06, 6.574359304067912e-06, 0.0007270937203429639, 0.0018232475267723203, 7.853259376133792e-06, 0.0003062742471229285, 0.0069750393740832806, 3.485084016574547e-05, 0.028292754665017128, 0.96165531873703, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0060739885084331036, 8.81520172697492e-05, 4.583776535582729e-05, 8.250482642324641e-05, 0.0001272044173674658, 0.000931251619476825, 0.0016700004925951362, 0.001938194502145052, 0.0021016972605139017, 0.01013167854398489, 0.016047311946749687, 0.0038353821728378534, 0.0122421495616436, 0.10037057101726532, 0.07853130996227264, 0.10997366905212402, 0.6558091044425964, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0021863009314984083, 1.1892925613210537e-05, 1.673595397733152e-05, 2.0815425159526058e-05, 2.1162606572033837e-05, 0.0007714940584264696, 0.0008652110700495541, 0.0005560303688980639, 0.00015655896277166903, 0.0019499914487823844, 0.01089120376855135, 0.00039543138700537384, 0.0015144728822633624, 0.06916314363479614, 0.004308109171688557, 0.004035164136439562, 0.49397027492523193, 0.40916597843170166, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.9797986169578508e-05, 4.972388296664576e-07, 9.16571174514047e-09, 1.3230415163434373e-07, 5.601821584377831e-08, 1.1903852126238235e-08, 6.153104550321586e-07, 4.0685499413939397e-08, 4.517450179264415e-06, 1.4017776265973225e-05, 1.6456850460144778e-07, 1.0863801662708283e-06, 7.444067705364432e-06, 1.0056634209831827e-06, 2.7594967832555994e-05, 0.0003137800667900592, 6.201119049364934e-06, 8.213330147555098e-06, 0.9995947480201721, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [6.497167760244338e-06, 8.692019264344708e-07, 1.4051418020244455e-06, 3.7704865007981425e-07, 3.3291723866568645e-07, 6.104985317278988e-08, 2.5430050953900718e-08, 1.1067811556131346e-07, 3.670877413242124e-05, 5.704819727725408e-07, 6.172358553158119e-07, 0.00015627963875886053, 0.00014737957098986953, 3.010378577528172e-06, 6.130677502369508e-05, 0.0017769928090274334, 1.8545953935245052e-05, 3.426316106924787e-05, 0.923351526260376, 0.07440314441919327, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0036490042693912983, 3.7494395655812696e-05, 1.7470769307692535e-05, 2.844917071342934e-05, 3.312426997581497e-05, 0.0001956923515535891, 0.0002647593501023948, 0.0002469044702593237, 0.00028033158741891384, 0.0013949198182672262, 0.0017910321475937963, 0.0004246800090186298, 0.001462371670641005, 0.009717759676277637, 0.008053947240114212, 0.013200577348470688, 0.06315969675779343, 0.07152792811393738, 0.0752311423420906, 0.024205099791288376, 0.7250776290893555, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0006410900386981666, 3.925704731955193e-06, 1.160737156169489e-06, 1.53754945131368e-06, 9.072649618246942e-07, 8.058219464146532e-06, 1.3223956557339989e-05, 4.619923402060522e-06, 3.678843131638132e-05, 4.565187919070013e-05, 7.577830547234043e-05, 2.2741753127775155e-05, 5.6684628361836076e-05, 0.00047375037684105337, 0.00036407698644325137, 0.00042400264646857977, 0.0037573366425931454, 0.00436312984675169, 0.0073952204547822475, 0.0050226240418851376, 0.05575619637966156, 0.9215313792228699, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005377684719860554, 8.12255109394755e-07, 2.301862195963622e-06, 1.4700369774800492e-06, 1.8703678961173864e-06, 6.024504273227649e-07, 4.127828560740454e-06, 1.6705761254343088e-06, 2.0471957213885617e-06, 7.430038385791704e-05, 3.5942291560786543e-06, 7.171507149905665e-06, 3.052820102311671e-05, 1.8927697965409607e-05, 0.0002543879672884941, 0.00019496382446959615, 0.0001262654986931011, 0.00018230534624308348, 0.0037352212239056826, 0.0006727118161506951, 0.0015798314707353711, 0.011177673004567623, 0.9813893437385559, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [8.512815838912502e-05, 4.870870498052682e-07, 1.2458991704988875e-06, 2.0811216927540954e-07, 1.9242862947521644e-07, 3.704045425934055e-08, 4.5552522465186485e-07, 5.504556455093734e-08, 1.532037913420936e-06, 5.0832995839300565e-06, 2.1400938976512407e-07, 1.4595335642297869e-06, 1.4768374057894107e-05, 1.0635013723003794e-06, 2.9461563826771453e-05, 5.728821997763589e-05, 6.615699930989649e-06, 7.84420899435645e-06, 0.00015063813771121204, 0.00022636981157120317, 7.97358006821014e-05, 0.00013040847261436284, 0.010868429206311703, 0.9883312582969666, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004166328872088343, 2.250376383017283e-06, 5.270549081615172e-06, 2.7549125661607832e-05, 3.963054405176081e-05, 7.51475135984947e-06, 8.100956620182842e-06, 1.1581355465750676e-05, 1.3205979485064745e-05, 6.750723696313798e-05, 2.5212744731106795e-05, 1.4860716873954516e-05, 7.521710358560085e-05, 0.00011457453365437686, 0.0002574536483734846, 0.00036668640677817166, 0.0006904755719006062, 0.0014518477255478501, 0.0032460198272019625, 0.0006353416829369962, 0.008745572529733181, 0.015190850012004375, 0.018150221556425095, 0.13303311169147491, 0.8174034953117371, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.6739300917834044e-05, 8.245349647495459e-08, 3.25739257789337e-08, 2.8737451529536884e-08, 1.621771161808283e-07, 1.5275413156601303e-09, 2.3278733607412505e-08, 5.576739070534131e-09, 1.2825815076666913e-07, 1.3429512080165296e-07, 6.160584486991638e-09, 2.5232574785150064e-07, 3.205763050573296e-06, 2.6277227505033807e-08, 7.619977395734168e-07, 7.901974640844855e-06, 1.3940466203621327e-07, 1.955263257968909e-07, 0.00032022615778259933, 8.173701644409448e-07, 1.5630049574610894e-06, 3.763174163395888e-06, 0.0007450517732650042, 0.003715225262567401, 3.9344242395600304e-05, 0.995144248008728, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00023842290102038532, 5.040115865995176e-06, 1.3208579048296087e-06, 8.59982810652582e-06, 5.166068604012253e-06, 1.3470912563207094e-07, 6.710822617606027e-07, 2.699698029573483e-07, 6.345725523715373e-06, 4.4178395910421386e-05, 2.7444988859315345e-07, 2.0171175492578186e-05, 1.652665923757013e-05, 8.12277562545205e-07, 0.0001077101260307245, 0.0001239744306076318, 3.50758546119323e-06, 8.505824553139973e-06, 6.980268517509103e-05, 0.00026813664590008557, 2.8792866942239925e-05, 0.0001195787190226838, 0.00012961654283571988, 0.0074259378015995026, 0.0007109702564775944, 0.005815016105771065, 0.9848405718803406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0004089944122824818, 6.095360731706023e-07, 1.9963858903793152e-06, 9.31032445805613e-06, 3.7690474528062623e-06, 1.987654741242295e-06, 4.576543801704247e-07, 1.0690715726013877e-06, 3.0871749459038256e-06, 9.211400538333692e-06, 4.724366590380669e-06, 1.2070000821040594e-06, 5.0956123232026584e-06, 1.8047107005259022e-05, 4.857866224483587e-05, 4.1693758248584345e-05, 0.00010537516209296882, 0.00022466831433121115, 0.0001152861732407473, 0.0002492887433618307, 0.0013673140201717615, 0.0016190309543162584, 0.0013275147648528218, 0.0034917735029011965, 0.06305646896362305, 0.006469056475907564, 0.09512638300657272, 0.8262879848480225, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00031804663012735546, 1.4181827054926543e-06, 3.185497519098135e-07, 2.782305045911926e-06, 5.388598219724372e-05, 4.300602185480784e-08, 5.386270345297817e-07, 8.497839587562339e-08, 5.999697236802604e-07, 4.11033352065715e-06, 7.099411902800057e-08, 8.454306907879072e-07, 2.52661493505002e-06, 2.1983862552588107e-07, 6.098245194152696e-06, 2.1129964807187207e-05, 9.411783707946597e-07, 9.950375670086942e-07, 3.041357012989465e-05, 2.976988980663009e-06, 8.453843292954843e-06, 2.6006960979430005e-05, 0.00023055804194882512, 0.0013383382465690374, 0.0002175823610741645, 0.0070143877528607845, 0.017924053594470024, 0.0005302667850628495, 0.9722621440887451, 0.0, 0.0, 0.0, 0.0], [1.850532953540096e-07, 5.565691907349901e-09, 7.262697110377303e-10, 7.745997621100287e-09, 1.507950919688028e-08, 8.228505654628293e-10, 1.7915414707303512e-09, 1.7345678227087546e-09, 1.0962554419791104e-08, 5.0032742393568697e-08, 3.922823399449271e-09, 1.0013942386422059e-07, 2.333783477581619e-08, 1.6555285142771936e-08, 5.171338912646206e-08, 1.6146054804266896e-07, 9.827900271375256e-08, 2.7074517561231914e-07, 5.411981874203775e-06, 2.508395482436754e-07, 1.1805932444985956e-06, 2.800419906634488e-06, 4.6103800741548184e-06, 0.00022679251560475677, 5.794241587864235e-05, 0.0004519554495345801, 0.00011976512178080156, 0.00023980850528459996, 0.0018740080995485187, 0.9970144033432007, 0.0, 0.0, 0.0], [4.347461799625307e-05, 2.9688746963074664e-06, 3.9193761836031626e-07, 9.117072750086663e-07, 4.364493406683323e-07, 4.686599552883308e-09, 9.240376641628245e-08, 1.241937130913584e-08, 7.188949524561394e-08, 6.304548492153117e-07, 6.1088996083924485e-09, 1.998713941020469e-07, 1.4578700984202442e-06, 1.4727314479046072e-08, 1.7957374609522958e-07, 2.6422709197504446e-05, 5.112656253913883e-08, 4.710664569529399e-08, 0.001576117007061839, 2.19894513975305e-06, 3.3339819083266775e-07, 2.1814748834003694e-06, 0.0001624040596652776, 0.001777765224687755, 1.1749206350941677e-05, 0.0036160899326205254, 6.199481867952272e-05, 3.516822835081257e-05, 0.001148868352174759, 0.016085589304566383, 0.975442111492157, 0.0, 0.0], [0.0011562916915863752, 6.199030394782312e-06, 6.137777290859958e-07, 3.102657274212106e-06, 8.228423666878371e-07, 4.3660193682626414e-08, 9.36323729661126e-08, 2.7735492125202654e-08, 5.109519634061144e-07, 1.2355296803434612e-06, 3.825297412163309e-08, 1.1272264828221523e-06, 1.9008897425010218e-06, 8.71404068902848e-08, 5.398183020588476e-06, 7.393429768853821e-06, 3.128866410406772e-07, 3.014625349351263e-07, 0.00013595313066616654, 1.334382250206545e-05, 2.399643108219607e-06, 3.871273293043487e-05, 0.0008219628944061697, 0.0006131263799034059, 2.151845910702832e-05, 0.004140724893659353, 0.000876085483469069, 0.00011684603668982163, 0.0021590653341263533, 0.02752523496747017, 0.04434969276189804, 0.9179997444152832, 0.0], [0.0031942895147949457, 4.681670179706998e-05, 3.499126614769921e-05, 2.7043073714594357e-05, 1.4000331248098519e-05, 1.708515628706664e-05, 8.05188301455928e-06, 3.909829956683097e-06, 1.0716935321397614e-05, 1.7930791727849282e-05, 1.4207519598130602e-05, 2.691129884624388e-05, 2.6412039005663246e-05, 3.110016041318886e-05, 1.1198587344551925e-05, 5.485498331836425e-05, 0.0001147448638221249, 0.00010493671288713813, 0.00028063019271939993, 0.00020839067292399704, 0.0009329256135970354, 0.0018990009557455778, 0.0011267588706687093, 0.0019257242092862725, 0.003917149268090725, 0.007499468047171831, 0.010695923119783401, 0.023199845105409622, 0.0303981751203537, 0.12729336321353912, 0.04325324296951294, 0.1766805350780487, 0.5669296979904175]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.25308626890182495, 0.746913731098175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3067159950733185, 0.32906362414360046, 0.36422038078308105, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07416975498199463, 0.16189664602279663, 0.054325900971889496, 0.709607720375061, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1668824404478073, 0.039017848670482635, 0.03822462633252144, 0.2139836996793747, 0.54189133644104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19483648240566254, 0.21119245886802673, 0.05150562524795532, 0.08703834563493729, 0.2299954891204834, 0.2254316210746765, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13900142908096313, 0.029748622328042984, 0.03860694169998169, 0.05133270099759102, 0.19284217059612274, 0.0801236629486084, 0.46834442019462585, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08969784528017044, 0.040806014090776443, 0.03473310545086861, 0.08414530009031296, 0.09911046922206879, 0.07059444487094879, 0.1361658126115799, 0.4447470009326935, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.059865228831768036, 0.020190464332699776, 0.018785344436764717, 0.10584733635187149, 0.05794881284236908, 0.02751772478222847, 0.0566631518304348, 0.08226758986711502, 0.5709143877029419, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.010379276238381863, 0.004698258358985186, 0.004143992904573679, 0.007291414309293032, 0.00625669676810503, 0.003318048780784011, 0.009175026789307594, 0.018754392862319946, 0.033931490033864975, 0.9020515084266663, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07270117104053497, 0.04776482656598091, 0.010500346310436726, 0.016159439459443092, 0.046178195625543594, 0.052494555711746216, 0.051845733076334, 0.21189497411251068, 0.0351581908762455, 0.17267192900180817, 0.2826305329799652, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015126904472708702, 0.004731189459562302, 0.0023226349148899317, 0.006575745530426502, 0.018362440168857574, 0.00333966501057148, 0.008784228004515171, 0.007409749086946249, 0.006289807613939047, 0.07638952881097794, 0.012003489769995213, 0.8386646509170532, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03233196586370468, 0.010628962889313698, 0.0026151672936975956, 0.0011762940557673573, 0.003093285718932748, 0.001505555002950132, 0.007079724222421646, 0.0028344697784632444, 0.005003892816603184, 0.012032603845000267, 0.003987777512520552, 0.051872458308935165, 0.8658378720283508, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.050455909222364426, 0.02653883397579193, 0.0057832710444927216, 0.00808730162680149, 0.021799396723508835, 0.02567034400999546, 0.022779053077101707, 0.09139904379844666, 0.0150173744186759, 0.07093929499387741, 0.12448637187480927, 0.1266833245754242, 0.06102132424712181, 0.3493391275405884, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.054741207510232925, 0.05296843498945236, 0.00398813234642148, 0.012351625598967075, 0.004415620118379593, 0.003596278140321374, 0.011385705322027206, 0.009828059002757072, 0.014749731868505478, 0.07078830152750015, 0.011209186166524887, 0.051640573889017105, 0.19734397530555725, 0.02712925150990486, 0.4738638401031494, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007163064554333687, 0.004963844083249569, 0.002769226673990488, 0.001942442380823195, 0.0105447294190526, 0.0014144864398986101, 0.0036636607255786657, 0.0031499469187110662, 0.005481296218931675, 0.021614039316773415, 0.003922195639461279, 0.07935075461864471, 0.2247789353132248, 0.009166942909359932, 0.027402641251683235, 0.5926717519760132, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.034611087292432785, 0.015296343713998795, 0.0030951257795095444, 0.003899100935086608, 0.009989405050873756, 0.011446859687566757, 0.009189359843730927, 0.03356511890888214, 0.005761211737990379, 0.025581028312444687, 0.04384688660502434, 0.045213598757982254, 0.02055337466299534, 0.11962322145700455, 0.15673252940177917, 0.13173440098762512, 0.3298613131046295, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01725386455655098, 0.004397520329803228, 0.004968686029314995, 0.007064308505505323, 0.0066348896361887455, 0.006910950876772404, 0.01138689648360014, 0.01617850363254547, 0.020317746326327324, 0.020396806299686432, 0.02342209778726101, 0.01890913024544716, 0.04024201259016991, 0.06200672313570976, 0.11645368486642838, 0.07975102216005325, 0.16864466667175293, 0.37506043910980225, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024591172114014626, 0.0002491978812031448, 3.400468995096162e-05, 0.00021132979600224644, 0.00020066795696038753, 0.00012589256220962852, 0.00036506872856989503, 0.0003735977807082236, 0.00013798548025079072, 0.0004702215956058353, 0.0002715179871302098, 0.0034170825965702534, 0.0006049805670045316, 0.0005454609054140747, 0.0008937669917941093, 0.0015852153301239014, 0.0012150767724961042, 0.0028212692122906446, 0.9840185046195984, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00015282450476661325, 0.03892359882593155, 0.0004715202085208148, 0.0003731821780093014, 3.271803871029988e-05, 1.0635604667186271e-05, 3.824580744549166e-06, 1.3244709407445043e-05, 6.418931297957897e-05, 2.7344427508069202e-05, 2.8319991542957723e-05, 0.0022295683156698942, 0.0013866405934095383, 7.163533155107871e-05, 0.00014025569544173777, 0.008246442303061485, 0.00019678677199408412, 0.0002054669166682288, 0.8976382613182068, 0.049783769994974136, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.026458118110895157, 0.010410883463919163, 0.0021332574542611837, 0.0024560203310102224, 0.005542597267776728, 0.005954570136964321, 0.004195827059447765, 0.013451523147523403, 0.0023519506212323904, 0.009471971541643143, 0.015433426015079021, 0.014804961159825325, 0.006865771021693945, 0.037873122841119766, 0.04744938388466835, 0.04547279328107834, 0.10199175775051117, 0.08666390180587769, 0.06470183283090591, 0.10160760581493378, 0.3947088122367859, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021157890558242798, 0.0035484314430505037, 0.001744112349115312, 0.003509304253384471, 0.004201889969408512, 0.0025777784176170826, 0.00401585828512907, 0.00459899939596653, 0.003955155611038208, 0.006455841474235058, 0.005907896906137466, 0.003733163233846426, 0.00901340413838625, 0.013927978463470936, 0.029895804822444916, 0.020082319155335426, 0.03804439678788185, 0.0688309520483017, 0.0815289244055748, 0.038340240716934204, 0.1516026258468628, 0.48332706093788147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0014211714733392, 0.00040690190508030355, 0.000271442870143801, 0.0014869269216433167, 0.0006475160480476916, 0.00025671522598713636, 0.00027297684573568404, 0.0005059858085587621, 0.0001753750693751499, 0.0012059618020430207, 0.000590289942920208, 0.0004307403869461268, 0.0003699965600389987, 0.0013322854647412896, 0.0007594284252263606, 0.0025183132383972406, 0.0035999834071844816, 0.004756723064929247, 0.011892947368323803, 0.0034102213103324175, 0.013851113617420197, 0.059073347598314285, 0.8907636404037476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.005058860406279564, 0.0008823536336421967, 0.0015358267119154334, 0.001690985169261694, 0.001343871233984828, 0.0004949704161845148, 0.0006572246202267706, 0.0005521580460481346, 0.0005812093731947243, 0.0020545588340610266, 0.0008380856597796082, 0.001006747712381184, 0.0011300832265987992, 0.001613990287296474, 0.005022868048399687, 0.014576872810721397, 0.003714330494403839, 0.00468147499486804, 0.007913815788924694, 0.006778385955840349, 0.01229135412722826, 0.027732999995350838, 0.10214643180370331, 0.7957004904747009, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00400469871237874, 0.0004576973442453891, 0.0015645234379917383, 0.003137205494567752, 0.0019937469623982906, 0.0003789262264035642, 0.000495778804179281, 0.0005698021268472075, 0.001005206722766161, 0.002343227155506611, 0.0005772152217105031, 0.00167561462149024, 0.0030787396244704723, 0.00122852623462677, 0.005278333555907011, 0.006328058894723654, 0.0031517331954091787, 0.005319315940141678, 0.021383123472332954, 0.006147709675133228, 0.01233222521841526, 0.07402164489030838, 0.12114029377698898, 0.561111569404602, 0.16127514839172363, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0011286563239991665, 4.9889247748069465e-05, 4.129129956709221e-05, 9.249628783436492e-05, 5.513632277143188e-05, 3.861729055643082e-05, 5.592032539425418e-05, 0.00014896265929564834, 5.509219590749126e-06, 6.81743913446553e-05, 3.620445204433054e-05, 4.623965287464671e-05, 0.0010655033402144909, 5.91988064115867e-05, 7.787259528413415e-05, 0.00038421311182901263, 0.00011692447878886014, 0.00022428260126616806, 0.0005083988653495908, 0.00015628148685209453, 0.0003233148599974811, 0.0007086825207807124, 0.0009207419352605939, 0.0008559718844480813, 0.0044144499115645885, 0.9884169697761536, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0024299719370901585, 0.0036596376448869705, 0.0006344981375150383, 0.0006202662480063736, 0.00298767676576972, 0.0003476368437986821, 0.0005059164250269532, 0.0004114972543902695, 0.00021321592794265598, 0.0006509354570880532, 0.00039556928095407784, 0.16986460983753204, 0.002188899787142873, 0.0006812376668676734, 0.0008683762862347066, 0.008905144408345222, 0.00146385352127254, 0.0013394575798884034, 0.006495132576674223, 0.007638710085302591, 0.004619397222995758, 0.003161789383739233, 0.014200900681316853, 0.046396635472774506, 0.022786062210798264, 0.2193925380706787, 0.4771403968334198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015610335394740105, 0.0009648075792938471, 0.0010281833820044994, 0.007148518692702055, 0.005316654220223427, 0.0005432681646198034, 0.0007839899626560509, 0.00047082934179343283, 0.0022980368230491877, 0.005463099107146263, 0.000548307376448065, 0.0008661507745273411, 0.0009035557741299272, 0.0009251964511349797, 0.008375934325158596, 0.001447812537662685, 0.0020614895038306713, 0.0036933831870555878, 0.017008863389492035, 0.006138677708804607, 0.007041893899440765, 0.04128709435462952, 0.06922485679388046, 0.26617175340652466, 0.05024215579032898, 0.11445145308971405, 0.13764898478984833, 0.232334703207016, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02189529500901699, 0.0013514539459720254, 0.001372796599753201, 0.00515771983191371, 0.01093278732150793, 0.0004452219291124493, 0.0016178288497030735, 0.0005713255377486348, 0.0007700272253714502, 0.003093538573011756, 0.0003616342437453568, 0.0009329022141173482, 0.0019032384734600782, 0.0005254382849670947, 0.000703618279658258, 0.0015780642861500382, 0.0010200473479926586, 0.0009346726001240313, 0.005167574156075716, 0.0007855411968193948, 0.002879679435864091, 0.0063371374271810055, 0.04951413348317146, 0.021886898204684258, 0.030046822503209114, 0.06941603124141693, 0.0299538541585207, 0.061116188764572144, 0.6677285432815552, 0.0, 0.0, 0.0, 0.0], [0.0002946658350992948, 8.085112494882196e-05, 0.00011992642976110801, 0.00017154029046650976, 0.0003899007278960198, 8.630252705188468e-05, 5.253727795206942e-05, 6.616362952627242e-05, 0.00011378394265193492, 6.200093048391864e-05, 0.00011238027218496427, 0.0005867318250238895, 9.785001748241484e-05, 0.00021334877237677574, 0.00040444263140670955, 0.00047262405860237777, 0.00047307409113273025, 0.0005199522129260004, 0.0008571160142309964, 0.0002272017445648089, 0.0015976777067407966, 0.0014780642231926322, 0.0016501424834132195, 0.00933963991701603, 0.012792701832950115, 0.012606944888830185, 0.021659579128026962, 0.02886745147407055, 0.07784781605005264, 0.8267576694488525, 0.0, 0.0, 0.0], [0.0006396546377800405, 0.00019920250633731484, 0.00033784619881771505, 0.0003228618297725916, 0.0003883979225065559, 2.2511427232529968e-05, 0.00011559672566363588, 1.9683695427374914e-05, 8.749772678129375e-05, 0.00016675752704031765, 2.1353165720938705e-05, 0.0001825768849812448, 0.002879620995372534, 3.2708361686673015e-05, 6.355986988637596e-05, 0.0009078506263904274, 6.314196798484772e-05, 4.954505493515171e-05, 0.005010188557207584, 0.00021555754938162863, 0.00018923298921436071, 0.0002357984340051189, 0.0021191705018281937, 0.0034752420615404844, 0.0004957504570484161, 0.01267303992062807, 0.0026274591218680143, 0.0011926378356292844, 0.0280115008354187, 0.1421455293893814, 0.7951086163520813, 0.0, 0.0], [0.005492144729942083, 0.0014179610880091786, 6.310038588708267e-05, 0.0009282363462261856, 0.0004675868258345872, 9.701211820356548e-05, 5.6787015637382865e-05, 6.824053707532585e-05, 4.054802775499411e-05, 0.00021011488570366055, 6.534687418024987e-05, 0.00011957778770010918, 9.082331962417811e-05, 9.077344293473288e-05, 0.00013827496150042862, 0.0004122828249819577, 0.0001690058852545917, 0.00013570708688348532, 0.002428323496133089, 0.00020654767286032438, 0.0004959595971740782, 0.00016561977099627256, 0.001825400977395475, 0.0016808173386380076, 0.0010187349980697036, 0.015071477741003036, 0.0034083749633282423, 0.003384191542863846, 0.018865497782826424, 0.010037423111498356, 0.008259555324912071, 0.923088550567627, 0.0], [0.009386110119521618, 0.0017380572389811277, 0.0018091707024723291, 0.0014350239653140306, 0.0016812816029414535, 0.0019791284576058388, 0.0014143437147140503, 0.002112781163305044, 0.0012016964610666037, 0.0010467651300132275, 0.0012532476102933288, 0.0012125700013712049, 0.0006952424882911146, 0.0016850167885422707, 0.001290497719310224, 0.0014949421165511012, 0.003052463522180915, 0.006176403723657131, 0.003993154037743807, 0.002994519891217351, 0.008827606216073036, 0.009542387910187244, 0.010477349162101746, 0.021451426669955254, 0.06221357733011246, 0.026861026883125305, 0.033906422555446625, 0.11987605690956116, 0.05027468875050545, 0.026796843856573105, 0.031165694817900658, 0.10728637874126434, 0.4436681270599365]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10520488768815994, 0.8947951793670654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.039072077721357346, 0.002017219550907612, 0.9589106440544128, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015578621067106724, 0.0008392521413043141, 0.000697975920047611, 0.9828841686248779, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008856472559273243, 9.934288755175658e-06, 1.117484407586744e-05, 0.0003024703182745725, 0.9908199906349182, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.35444995760917664, 0.030589260160923004, 0.0598895363509655, 0.022903503850102425, 0.04747587814927101, 0.4846918284893036, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.044769056141376495, 0.0015664675738662481, 0.000377385294996202, 0.00025073246797546744, 0.00040889671072363853, 0.0002606563502922654, 0.952366828918457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09058351069688797, 0.0011470747413113713, 0.006027341354638338, 0.0005468469462357461, 0.001709421630948782, 0.0037850833032280207, 0.002684539183974266, 0.8935161828994751, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006010739598423243, 2.428906736895442e-05, 0.0003126677474938333, 1.868291656137444e-05, 0.00029792427085340023, 9.904524631565437e-06, 6.619560281251324e-06, 7.912205433058261e-07, 0.9933183193206787, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.004546831827610731, 0.00020019400108139962, 0.00029920271481387317, 0.0013363977195695043, 0.0003267655265517533, 8.741383226151811e-07, 1.7415770344086923e-05, 2.7834221327793784e-07, 0.00029665272450074553, 0.9929754734039307, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12262558192014694, 0.02392655797302723, 0.03827647864818573, 0.016171954572200775, 0.030366200953722, 0.29375946521759033, 0.10661349445581436, 0.07059947401285172, 0.04190007224678993, 0.01783308945596218, 0.23792754113674164, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0008121037972159684, 0.00011546615860424936, 0.00013500881323125213, 5.539281846722588e-05, 0.0001317410496994853, 1.0027158623415744e-06, 1.3799076441500802e-06, 1.6001582991975738e-07, 1.897293486763374e-06, 5.419645390247751e-07, 4.7530824076602585e-07, 0.9987448453903198, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013333045644685626, 0.00010533037857385352, 0.0007669609040021896, 0.0009021882433444262, 3.570836042854353e-06, 8.120483130369394e-07, 1.4038103302027594e-07, 9.56129397877703e-08, 1.6865070620042388e-06, 2.9789011023240164e-05, 4.381340659165289e-07, 1.196580001305847e-06, 0.9968544840812683, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08159806579351425, 0.02138884738087654, 0.02887715771794319, 0.012675785459578037, 0.024931589141488075, 0.2299650013446808, 0.10401234030723572, 0.06472096592187881, 0.034600917249917984, 0.015198715031147003, 0.1856415569782257, 0.020528052002191544, 0.005003851838409901, 0.17085717618465424, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0013488855911418796, 8.921194239519536e-05, 0.0006859959685243666, 0.0006963199120946229, 2.124857928720303e-05, 3.763386189348239e-07, 6.226725872693351e-07, 4.307211725063098e-07, 7.091866336850217e-06, 0.0013103662058711052, 1.6113735057388112e-07, 9.548742809784017e-07, 0.00016453575517516583, 1.1531913912676828e-07, 0.9956735968589783, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010867483215406537, 0.00040724663995206356, 0.00010700309940148145, 0.0006782697746530175, 0.00011028441076632589, 1.3184198621729593e-07, 2.660260634002043e-07, 4.619550111328863e-07, 8.409681868215557e-06, 0.0007784871850162745, 5.1094296793507965e-08, 8.792052312855958e-07, 4.934279786539264e-05, 3.489302713433062e-08, 0.00011441176320658997, 0.9966579675674438, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05868092179298401, 0.018785739317536354, 0.024067306891083717, 0.011013139970600605, 0.02145637758076191, 0.1911056935787201, 0.09902718663215637, 0.06065867841243744, 0.030162733048200607, 0.014801912941038609, 0.15370333194732666, 0.016615163534879684, 0.004385383799672127, 0.14085334539413452, 0.015015006065368652, 0.005390211008489132, 0.13427790999412537, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07323029637336731, 0.028374552726745605, 0.021058231592178345, 0.007611874025315046, 0.010053571313619614, 0.06553954631090164, 0.14559760689735413, 0.08620846271514893, 0.010241259820759296, 0.007425738964229822, 0.045262712985277176, 0.006621452979743481, 0.001162177650257945, 0.0392385758459568, 0.008256874047219753, 0.0030656878370791674, 0.0361226424574852, 0.40492871403694153, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [7.9444631410297e-05, 1.8382071402811562e-06, 3.713425300588824e-08, 1.5575264114886522e-05, 3.472369144219556e-06, 1.3165072365595165e-09, 2.7038657535172206e-08, 1.718662012706318e-08, 9.232245190560207e-08, 1.350632737739943e-05, 4.4787715203220557e-10, 2.0416692780855783e-09, 5.542018044479846e-08, 3.063855003038185e-10, 4.217707214593247e-07, 7.065867521305336e-06, 2.424434542014353e-10, 1.4397331105087119e-09, 0.9998784065246582, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0029345010407269, 0.0005883852718397975, 0.00425031129270792, 0.004028464201837778, 0.00011592968076001853, 4.331630407250486e-06, 6.773913810320664e-06, 7.076482143020257e-05, 0.00024952724925242364, 0.00031593386665917933, 2.0824440980504733e-06, 4.409225584822707e-05, 0.00030446963501162827, 1.47026048580301e-06, 0.00019761381554417312, 0.0013599277008324862, 1.2478676580940373e-06, 6.875189228594536e-06, 0.00019261166744399816, 0.9853248000144958, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04056535288691521, 0.017898328602313995, 0.020600980147719383, 0.009821098297834396, 0.017882810905575752, 0.15654006600379944, 0.09208227694034576, 0.05628824979066849, 0.026661338284611702, 0.014471273869276047, 0.12537625432014465, 0.01398677472025156, 0.003927926998585463, 0.11453613638877869, 0.013395383022725582, 0.004846665076911449, 0.10880250483751297, 0.04953395947813988, 0.005186410620808601, 0.004420871380716562, 0.10317537188529968, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.024536358192563057, 0.003124336479231715, 0.0009699302609078586, 4.631542833521962e-05, 0.00013434779248200357, 0.000645394145976752, 0.011668854393064976, 0.0003271344176027924, 0.00045730415149591863, 2.3976075681275688e-05, 0.00034454173874109983, 4.996423740522005e-05, 2.5757408366189338e-05, 0.0002821777598001063, 6.525324351969175e-06, 4.981804522685707e-06, 0.0002355400356464088, 0.00026645115576684475, 2.5311132048955187e-05, 2.9761094992863946e-05, 0.0001880924974102527, 0.9566069841384888, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.00207599182613194, 0.00014591730723623186, 5.2898929425282404e-05, 4.218065441818908e-05, 4.738710049423389e-05, 4.142319198763289e-07, 0.00012964261986780912, 2.4250950446003117e-05, 1.803399186428578e-06, 1.4839238247077446e-06, 1.448993742769744e-07, 4.199928298476152e-06, 4.106833273453958e-08, 1.0260460925337611e-07, 8.285439889732515e-07, 5.781897129963909e-07, 7.695144432773304e-08, 6.669583996199435e-08, 6.472608220065013e-05, 1.570085601088067e-06, 5.641819811330606e-08, 4.403018920129398e-06, 0.997401237487793, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005159112042747438, 0.00011814642493845895, 3.2705673220334575e-05, 0.00021188720711506903, 4.5635246351594105e-05, 2.9248869282128e-07, 3.057942421946791e-06, 7.607361141026558e-08, 0.00013753684470430017, 0.0005283545469865203, 1.157602511625555e-07, 4.028581315651536e-06, 2.0793006569874706e-06, 8.805425011360057e-08, 0.00010051687422674149, 4.4316235289443284e-05, 6.963711740581857e-08, 9.373040654736542e-08, 3.3377720683347434e-05, 1.5888888810877688e-05, 5.610685960277806e-08, 7.05264184830412e-08, 9.389279512106441e-06, 0.9981963038444519, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06915231794118881, 0.014032094739377499, 0.005599095486104488, 0.001257685013115406, 0.0008467998704873025, 0.00960117019712925, 0.008965770713984966, 0.05247688293457031, 0.0002502834831830114, 0.0009063859470188618, 0.005419899243861437, 0.0007309205830097198, 0.00019187942962162197, 0.004396578762680292, 0.005964715499430895, 0.0003558540192898363, 0.0038770136889070272, 0.009727222844958305, 0.001102348556742072, 0.00036442853161133826, 0.0034139559138566256, 0.0030523529276251793, 0.005698243156075478, 0.00029557303059846163, 0.7923206686973572, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010461577912792563, 1.955717925739009e-05, 1.7358993318339344e-06, 3.103052222286351e-05, 2.73080040642526e-05, 1.0180698062356441e-08, 4.815777288058598e-07, 1.7494264170636598e-07, 7.312424088468106e-08, 7.98307667082554e-07, 3.4643168334014263e-09, 4.906727099296404e-06, 1.0718952836441531e-07, 2.35255281921809e-09, 2.383409878348175e-07, 1.3731346371059772e-06, 1.828867168818249e-09, 6.247621975319362e-09, 9.208356459566858e-06, 1.6384318541895482e-06, 1.252392411998926e-09, 8.271274221094416e-10, 8.456785508315079e-06, 3.118682570857345e-06, 9.241420073635709e-09, 0.9988435506820679, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0010544945253059268, 0.00021373217168729752, 0.0002815722255036235, 0.0011373378802090883, 0.00014783968799747527, 2.00685326490202e-06, 4.831360456591938e-06, 3.48233857039304e-06, 5.100344878883334e-06, 0.000124204860185273, 9.301529075855797e-07, 1.058264297171263e-05, 1.1688286576827522e-05, 7.077518944242911e-07, 0.003411603858694434, 0.00019905316003132612, 5.995618153065152e-07, 4.643046622732072e-07, 1.9585138943511993e-05, 0.00011087782331742346, 4.7500583377768635e-07, 3.280884754985891e-07, 4.535358584689675e-06, 5.161714580026455e-05, 3.247476740853017e-07, 1.3071397916064598e-05, 0.9931889772415161, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02420073375105858, 0.0028201257809996605, 0.004319292027503252, 0.0009108288213610649, 0.0017753917491063476, 0.0038099121302366257, 0.002953318180516362, 0.01671474799513817, 0.0013931727735325694, 0.005003178026527166, 0.002276085549965501, 0.0005876405630260706, 0.0009901626035571098, 0.0018066922202706337, 0.0031312357168644667, 0.00023389342823065817, 0.0016082689398899674, 0.009910290129482746, 0.0028539495542645454, 0.0002049512550001964, 0.001371262944303453, 0.006918995641171932, 0.0010336557170376182, 0.0001452838332625106, 0.017489774152636528, 0.00046553026186302304, 0.00030240791966207325, 0.884769082069397, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0005256083677522838, 3.528641173033975e-06, 1.7037668840202969e-06, 9.408171172253788e-05, 0.6647970080375671, 1.6531461710656004e-07, 9.511094845038315e-07, 8.886875662028615e-07, 1.7016665879054926e-05, 3.742176340892911e-05, 6.832259913380767e-08, 9.810918527364265e-06, 7.548730707185314e-08, 4.963247590694664e-08, 2.0393890736158937e-06, 7.686546268814709e-06, 4.0668474809990585e-08, 2.6744292114244672e-08, 3.343185017001815e-05, 4.3410790340203675e-07, 3.007985327485585e-08, 3.206054088877863e-08, 4.900574822386261e-06, 3.0609396617364837e-06, 1.4603182663108782e-08, 2.2145519324112684e-05, 9.21115315577481e-06, 3.164275241829273e-08, 0.3344285786151886, 0.0, 0.0, 0.0, 0.0], [0.004676999989897013, 0.0002802134840749204, 8.346101822098717e-05, 3.157018227284425e-06, 4.065346672632586e-07, 1.3343500882001536e-07, 2.6423611416248605e-05, 1.1439977498639564e-07, 2.787237463053316e-07, 3.7496133700187784e-06, 4.6529063268963e-08, 1.721399939924595e-06, 1.8165634685374243e-07, 3.148475968828279e-08, 1.8334266371766716e-07, 5.214756271243459e-08, 2.5215808108214333e-08, 3.078434929193463e-08, 7.142043045860191e-08, 1.6640171907056356e-08, 1.7768222448921733e-08, 8.63450821952938e-08, 2.8009644665871747e-06, 8.454135809188301e-07, 3.098357126418705e-08, 2.927866091795295e-07, 5.676233172380307e-07, 5.055421414823513e-09, 2.9141034474378102e-08, 0.9949179887771606, 0.0, 0.0, 0.0], [5.6368040532106534e-05, 0.00020023771503474563, 1.6319127098540775e-06, 1.5089342468854738e-06, 1.777980287442915e-05, 5.956442006294083e-10, 6.940383173059672e-06, 5.034020311711629e-09, 2.168291644011333e-07, 1.7164308019346208e-06, 1.9326304445677067e-10, 1.8911462618120822e-08, 9.53887777654927e-08, 1.2946910210587248e-10, 8.07718159023807e-09, 4.3866159103345126e-05, 9.768721942471359e-11, 5.256080570781307e-10, 6.93218462402001e-05, 7.505895638360016e-08, 7.06931874150385e-11, 3.988054686487885e-08, 1.061030798155116e-05, 2.212075060015195e-06, 8.688650909860485e-10, 9.928508006851189e-06, 5.390095036972298e-09, 1.1354502754690188e-10, 3.0600494937971234e-06, 3.8945665892242687e-07, 0.9995740056037903, 0.0, 0.0], [0.0003556977608241141, 1.9489476471790113e-05, 9.160975423583295e-06, 1.3085264072287828e-05, 2.3808899641153403e-05, 2.9718961513935938e-08, 4.1605944716138765e-05, 1.775650702029452e-07, 6.896241870890663e-07, 1.0506230410101125e-06, 1.2230765733534099e-08, 4.29055216955021e-06, 3.295830310889869e-07, 9.21108878060295e-09, 1.7466412600697367e-06, 4.682785402110312e-06, 7.44272554698e-09, 9.673800427378865e-09, 3.9665097574470565e-05, 7.324884450099489e-07, 5.543690839715509e-09, 1.0396706073834139e-07, 1.4401590306079015e-05, 8.873348633642308e-06, 5.98941607421466e-09, 1.5627783795935102e-05, 2.9811219519615406e-06, 6.874355751307348e-09, 4.8546603466093075e-06, 7.072258540574694e-06, 6.210679657669971e-06, 0.999423623085022, 0.0], [0.054552722722291946, 0.016429070383310318, 0.01600833795964718, 0.00698688393458724, 0.02488415688276291, 0.07376936078071594, 0.07933781296014786, 0.05594553053379059, 0.027268217876553535, 0.013368784449994564, 0.055903103202581406, 0.007223032880574465, 0.005426581017673016, 0.05012327805161476, 0.00827906746417284, 0.003060963936150074, 0.046822961419820786, 0.04751850664615631, 0.0028449436649680138, 0.0070379748940467834, 0.04282716289162636, 0.0513819120824337, 0.027282366529107094, 0.026375338435173035, 0.026159213855862617, 0.01199798472225666, 0.006220122333616018, 0.03648781031370163, 0.011124703101813793, 0.0052185626700520515, 0.0025591999292373657, 0.002604027511551976, 0.14697042107582092]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9192656874656677, 0.08073429763317108, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.454373836517334, 0.42650434374809265, 0.11912187188863754, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5267521142959595, 0.22321633994579315, 0.12420625239610672, 0.12582530081272125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.39034876227378845, 0.1709940880537033, 0.06872706860303879, 0.1837182343006134, 0.18621177971363068, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1974422186613083, 0.25436854362487793, 0.14488446712493896, 0.18925854563713074, 0.20821942389011383, 0.005826781038194895, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19473576545715332, 0.13241834938526154, 0.1439192295074463, 0.1512073427438736, 0.30024468898773193, 0.02058463543653488, 0.05688997730612755, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1269667148590088, 0.2159738838672638, 0.16089318692684174, 0.1969883292913437, 0.1884462833404541, 0.013544922694563866, 0.07520707696676254, 0.021979689598083496, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.291212260723114, 0.05954229086637497, 0.1107940748333931, 0.09717852622270584, 0.22345606982707977, 0.016262579709291458, 0.08719443529844284, 0.024762297049164772, 0.0895974338054657, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.22981956601142883, 0.03802676126360893, 0.1601382941007614, 0.11780411005020142, 0.161811962723732, 0.035528186708688736, 0.07561112940311432, 0.020756037905812263, 0.10670629143714905, 0.05379772558808327, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1331443041563034, 0.18858659267425537, 0.10808098316192627, 0.1599496752023697, 0.17739209532737732, 0.0035980113316327333, 0.0250620786100626, 0.01286292914301157, 0.0745309591293335, 0.11275246739387512, 0.004039983730763197, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1674639731645584, 0.1435522586107254, 0.061185698956251144, 0.17358073592185974, 0.10113828629255295, 0.017553461715579033, 0.032395295798778534, 0.007995499297976494, 0.05453450232744217, 0.0788319781422615, 0.017258206382393837, 0.14451003074645996, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2699682116508484, 0.1302906721830368, 0.06605684012174606, 0.06156144291162491, 0.08437127619981766, 0.019846610724925995, 0.016312766820192337, 0.012965915724635124, 0.032314639538526535, 0.05254792422056198, 0.01986861042678356, 0.13695642352104187, 0.09693863987922668, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09820349514484406, 0.13618648052215576, 0.080057792365551, 0.12152864038944244, 0.1357937604188919, 0.0024323551915585995, 0.01778949424624443, 0.009086495265364647, 0.05637528374791145, 0.08521639555692673, 0.002753337612375617, 0.1520378738641739, 0.09946724772453308, 0.003071404527872801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1701289266347885, 0.08885715156793594, 0.0908968597650528, 0.08085789531469345, 0.11427337676286697, 0.010901493020355701, 0.014788409695029259, 0.005497371777892113, 0.05151258036494255, 0.05330684408545494, 0.010595816187560558, 0.0944228321313858, 0.1309811770915985, 0.011153677478432655, 0.07182560116052628, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10365856438875198, 0.03588540107011795, 0.08803563565015793, 0.060879383236169815, 0.08348328620195389, 0.030805740505456924, 0.02961592748761177, 0.010918696410953999, 0.040499936789274216, 0.04288606718182564, 0.03219295293092728, 0.1500813364982605, 0.12658777832984924, 0.03490216284990311, 0.058254316449165344, 0.0713127851486206, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07047095894813538, 0.09600995481014252, 0.05621575936675072, 0.08699886500835419, 0.09934128075838089, 0.0016298795817419887, 0.0124086057767272, 0.0063804336823523045, 0.04101405292749405, 0.06348713487386703, 0.0018482196610420942, 0.11387647688388824, 0.07328980416059494, 0.0020712942350655794, 0.07246169447898865, 0.2001277208328247, 0.0023677784483879805, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05659443140029907, 0.08906207978725433, 0.047032371163368225, 0.09457873553037643, 0.08793891966342926, 0.001972165424376726, 0.017767686396837234, 0.005990440025925636, 0.06014859303832054, 0.08834967017173767, 0.002263767644762993, 0.10461527854204178, 0.06079372763633728, 0.0025566222611814737, 0.07818630337715149, 0.19496017694473267, 0.002929988317191601, 0.004259059671312571, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15970627963542938, 0.026096465066075325, 0.045151788741350174, 0.034975819289684296, 0.08030673116445541, 0.054690372198820114, 0.04078543931245804, 0.026513027027249336, 0.027251916006207466, 0.03919815644621849, 0.05351432412862778, 0.048429232090711594, 0.0626426711678505, 0.055582042783498764, 0.05559665709733963, 0.04792958125472069, 0.05676328018307686, 0.04892297461628914, 0.03594321757555008, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08663123100996017, 0.09557438641786575, 0.03673810511827469, 0.06946706771850586, 0.05681459605693817, 0.00819020252674818, 0.022195840254426003, 0.007642821874469519, 0.03146827220916748, 0.07555200904607773, 0.008965258486568928, 0.052634816616773605, 0.04491107165813446, 0.009935041889548302, 0.04481375962495804, 0.15198467671871185, 0.010933407582342625, 0.007993672974407673, 0.16402563452720642, 0.013528095558285713, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.047834545373916626, 0.06669358164072037, 0.03798915818333626, 0.06040366366505623, 0.06899041682481766, 0.0010267285397276282, 0.008172675967216492, 0.004158171359449625, 0.02826826274394989, 0.044072408229112625, 0.0011586480541154742, 0.0783291608095169, 0.05082903057336807, 0.0013004965148866177, 0.049042973667383194, 0.1414499431848526, 0.0014934338396415114, 0.0025566837284713984, 0.17534083127975464, 0.12910206615924835, 0.0017870559822767973, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04114091023802757, 0.047660764306783676, 0.035869985818862915, 0.06666103005409241, 0.07969272136688232, 0.002365076681599021, 0.02034018374979496, 0.004854497499763966, 0.0643620565533638, 0.06737154722213745, 0.002707371022552252, 0.05596054345369339, 0.05648666247725487, 0.0030767249409109354, 0.053543876856565475, 0.15165922045707703, 0.003539715427905321, 0.004673928488045931, 0.1640271544456482, 0.059681519865989685, 0.004154941998422146, 0.010169517248868942, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0662880688905716, 0.04092542454600334, 0.027322672307491302, 0.04505367577075958, 0.06575214862823486, 0.015657566487789154, 0.02788187749683857, 0.013711252249777317, 0.03174295648932457, 0.05280744284391403, 0.015447776764631271, 0.03218112140893936, 0.03290105611085892, 0.01629885658621788, 0.0266366396099329, 0.09772849828004837, 0.01711421273648739, 0.016919447109103203, 0.09847065061330795, 0.04787907376885414, 0.018412543460726738, 0.015331602655351162, 0.17753544449806213, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09801430255174637, 0.026328813284635544, 0.03586812689900398, 0.0571306012570858, 0.06943874806165695, 0.01212895568460226, 0.020909082144498825, 0.008747957646846771, 0.02699965611100197, 0.037266768515110016, 0.011716341599822044, 0.08499541133642197, 0.05501337721943855, 0.012196706607937813, 0.035639096051454544, 0.08789657056331635, 0.012970106676220894, 0.010851009748876095, 0.06408388167619705, 0.059639204293489456, 0.01449217926710844, 0.006093730218708515, 0.09761444479227066, 0.053964968770742416, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.027693238109350204, 0.04073774442076683, 0.031223520636558533, 0.05879037827253342, 0.056018270552158356, 0.001571478322148323, 0.011908039450645447, 0.003339792601764202, 0.05176553130149841, 0.052912451326847076, 0.0018259811913594604, 0.06030035391449928, 0.028357183560729027, 0.002087807282805443, 0.0508648119866848, 0.07898834347724915, 0.0023987069725990295, 0.0029048253782093525, 0.11135730892419815, 0.0458601638674736, 0.0028849190566688776, 0.0114312544465065, 0.12340553104877472, 0.1348847597837448, 0.006487558595836163, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15390774607658386, 0.023786790668964386, 0.053202442824840546, 0.03963885456323624, 0.057110611349344254, 0.015628935769200325, 0.020206071436405182, 0.009252365678548813, 0.021311862394213676, 0.029029956087470055, 0.014741908758878708, 0.0612383671104908, 0.10112590342760086, 0.015157821588218212, 0.04335403069853783, 0.04090717062354088, 0.015732670202851295, 0.02007340081036091, 0.053749360144138336, 0.034686144441366196, 0.016989383846521378, 0.0063684964552521706, 0.04042018949985504, 0.031060416251420975, 0.013937166891992092, 0.06738188862800598, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08555091172456741, 0.0350574254989624, 0.05885253846645355, 0.034979090094566345, 0.051787421107292175, 0.002511989790946245, 0.004151217173784971, 0.002085915533825755, 0.014081131666898727, 0.024491365998983383, 0.0023760052863508463, 0.0953512117266655, 0.07766136527061462, 0.0024908813647925854, 0.04572677239775658, 0.05774112790822983, 0.0026603792794048786, 0.0033477547112852335, 0.04799828678369522, 0.02678881399333477, 0.002958717057481408, 0.004655787255614996, 0.03607098385691643, 0.02159917913377285, 0.004788634832948446, 0.2205219715833664, 0.033713094890117645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.020706715062260628, 0.03719407320022583, 0.01972370035946369, 0.038942232728004456, 0.03182700648903847, 0.0012253677705302835, 0.008815756998956203, 0.00240529328584671, 0.0345795638859272, 0.04729808494448662, 0.0013953752350062132, 0.05148209258913994, 0.020486755296587944, 0.0015608404064550996, 0.03826119378209114, 0.04863752797245979, 0.0017825323157012463, 0.002544606104493141, 0.06541350483894348, 0.04188564047217369, 0.002096060197800398, 0.009401391260325909, 0.08460026234388351, 0.09019990265369415, 0.005403279792517424, 0.22932566702365875, 0.056922730058431625, 0.005882919300347567, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08582808822393417, 0.03967176377773285, 0.014841809868812561, 0.056536201387643814, 0.0596061572432518, 0.006987510249018669, 0.010110095143318176, 0.002887724433094263, 0.0153228510171175, 0.02592874877154827, 0.006830442231148481, 0.03354894742369652, 0.02109353430569172, 0.007213153410702944, 0.01932338811457157, 0.05298227444291115, 0.007658612914383411, 0.00515784090384841, 0.06523182988166809, 0.018170002847909927, 0.008534705266356468, 0.004368167836219072, 0.026912428438663483, 0.049627941101789474, 0.0044043343514204025, 0.19388671219348907, 0.03933226317167282, 0.005363547708839178, 0.11263900995254517, 0.0, 0.0, 0.0, 0.0], [0.07040350884199142, 0.05926162004470825, 0.0153348408639431, 0.02413902059197426, 0.015728846192359924, 0.019492097198963165, 0.017311522737145424, 0.017303062602877617, 0.02056325040757656, 0.045155834406614304, 0.021815847605466843, 0.07079730927944183, 0.030063951388001442, 0.02354060858488083, 0.020684486255049706, 0.021163659170269966, 0.02644512802362442, 0.02354004792869091, 0.0566372387111187, 0.033694788813591, 0.03070729598402977, 0.015993362292647362, 0.015474083833396435, 0.030803611502051353, 0.03180558234453201, 0.07969608902931213, 0.034374866634607315, 0.03453716263175011, 0.030624372884631157, 0.06290693581104279, 0.0, 0.0, 0.0], [0.05338500067591667, 0.029621543362736702, 0.021030088886618614, 0.023695707321166992, 0.03343639522790909, 0.018393810838460922, 0.02406071498990059, 0.022810013964772224, 0.01397236343473196, 0.028813544660806656, 0.01905974932014942, 0.015371972694993019, 0.025335317477583885, 0.019971944391727448, 0.014075741171836853, 0.0388445146381855, 0.021346211433410645, 0.019321419298648834, 0.05632662773132324, 0.02136000245809555, 0.023394575342535973, 0.023136645555496216, 0.06416380405426025, 0.06724687665700912, 0.03469511494040489, 0.043490227311849594, 0.03032725118100643, 0.03285713493824005, 0.05327510088682175, 0.06599811464548111, 0.0411825068295002, 0.0, 0.0], [0.18177089095115662, 0.019533155485987663, 0.028002919629216194, 0.02911021187901497, 0.04134832322597504, 0.01801947131752968, 0.010471729561686516, 0.008382909931242466, 0.015335792675614357, 0.021839451044797897, 0.01689334399998188, 0.038061413913965225, 0.03643865883350372, 0.017464246600866318, 0.026129068806767464, 0.040115032345056534, 0.018317226320505142, 0.012488218024373055, 0.05136968195438385, 0.01944384165108204, 0.019889770075678825, 0.003576454007998109, 0.026232969015836716, 0.01842918060719967, 0.01172375027090311, 0.032729875296354294, 0.0348082073032856, 0.01203504391014576, 0.060302987694740295, 0.007873310707509518, 0.042272936552762985, 0.07958995550870895, 0.0], [0.01500341109931469, 0.020334843546152115, 0.00781343039125204, 0.01655357889831066, 0.016466103494167328, 0.00023809671984054148, 0.0019598952494561672, 0.00068982521770522, 0.006483219563961029, 0.010734076611697674, 0.0002443529083393514, 0.01463468000292778, 0.009247624315321445, 0.00026059168158099055, 0.00933992862701416, 0.04272656887769699, 0.00028863895568065345, 0.0003811671631410718, 0.05474938452243805, 0.0945277139544487, 0.0003421129658818245, 0.001727021881379187, 0.04745931923389435, 0.026613786816596985, 0.001023866469040513, 0.16537979245185852, 0.014187128283083439, 0.001309234299696982, 0.04770652949810028, 0.011198495514690876, 0.2205076813697815, 0.13942602276802063, 0.0004418623575475067]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9333658814430237, 0.06663414090871811, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.32452353835105896, 0.5923718810081482, 0.08310462534427643, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14159762859344482, 0.5143808126449585, 0.2814413607120514, 0.0625801756978035, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2526225745677948, 0.05834067985415459, 0.10458880662918091, 0.4411161541938782, 0.14333176612854004, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.18051564693450928, 0.09828895330429077, 0.10873967409133911, 0.12262701243162155, 0.0999554842710495, 0.3898731768131256, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17192299664020538, 0.023320984095335007, 0.12332724779844284, 0.06076502054929733, 0.05971726402640343, 0.34759145975112915, 0.21335503458976746, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09627045691013336, 0.04683617129921913, 0.028257353231310844, 0.032392505556344986, 0.05865046754479408, 0.15829336643218994, 0.30996477603912354, 0.2693348824977875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09948232769966125, 0.06464184820652008, 0.028671858832240105, 0.07305678725242615, 0.029787244275212288, 0.13960032165050507, 0.20754693448543549, 0.26580217480659485, 0.09141051024198532, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.05632041022181511, 0.029885882511734962, 0.013419928960502148, 0.0024075538385659456, 0.01324548665434122, 0.04564536362886429, 0.05522561073303223, 0.10027098655700684, 0.65799880027771, 0.02558007836341858, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04266997054219246, 0.010426478460431099, 0.010652223601937294, 0.013446640223264694, 0.009353154338896275, 0.04015978053212166, 0.07105177640914917, 0.13737650215625763, 0.10423263162374496, 0.21736650168895721, 0.3432643711566925, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.029683876782655716, 0.002561642322689295, 0.004407485947012901, 0.034248728305101395, 0.014238744042813778, 0.028001848608255386, 0.022288037464022636, 0.0710596889257431, 0.040262337774038315, 0.5543042421340942, 0.17333032190799713, 0.0256129652261734, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.040395937860012054, 0.008631578646600246, 0.0054942285642027855, 0.0038641411811113358, 0.02336312271654606, 0.02720699831843376, 0.025677325204014778, 0.05048522353172302, 0.05105609819293022, 0.050417814403772354, 0.14164336025714874, 0.5115285515785217, 0.06023569777607918, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02870640903711319, 0.004808461759239435, 0.004723651800304651, 0.005382603034377098, 0.00345604564063251, 0.014470165595412254, 0.024037357419729233, 0.0416262112557888, 0.035672929137945175, 0.07060731947422028, 0.10830993950366974, 0.058116037398576736, 0.20621228218078613, 0.39387065172195435, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.053426485508680344, 0.011923287995159626, 0.015572269447147846, 0.0021123234182596207, 0.005851047113537788, 0.022170916199684143, 0.03818468004465103, 0.02741158753633499, 0.05464285984635353, 0.04910825192928314, 0.12328367680311203, 0.05287177115678787, 0.07545658200979233, 0.3701530694961548, 0.09783123433589935, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008346711285412312, 0.00027852741186507046, 0.0011786019895225763, 0.002139552729204297, 0.0024761699605733156, 0.002918289741501212, 0.004576897248625755, 0.003965601325035095, 0.004973019938915968, 0.011347777210175991, 0.013014056719839573, 0.005545969121158123, 0.02089557610452175, 0.03838542848825455, 0.8750316500663757, 0.004926190245896578, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.02170032635331154, 0.002678667427971959, 0.0023839266505092382, 0.0028156302869319916, 0.001556233735755086, 0.006139400415122509, 0.009039802476763725, 0.013740871101617813, 0.012412197887897491, 0.024686532095074654, 0.034069038927555084, 0.017561456188559532, 0.06082957983016968, 0.12004053592681885, 0.11516799032688141, 0.12663696706295013, 0.42854082584381104, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.019681395962834358, 0.002866192953661084, 0.002752866828814149, 0.0023642033338546753, 0.0022205759305506945, 0.004776967689394951, 0.0050153969787061214, 0.007020784076303244, 0.0066029904410243034, 0.015824900940060616, 0.021779099479317665, 0.02237587794661522, 0.03905551880598068, 0.07342638075351715, 0.0722755640745163, 0.08103632181882858, 0.25563210248947144, 0.3652927577495575, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03443868085741997, 0.002429508836939931, 0.0027038808912038803, 0.0010108926799148321, 0.0028212431352585554, 0.009310662746429443, 0.007474060636013746, 0.010526387952268124, 0.019342582672834396, 0.006583398208022118, 0.030525239184498787, 0.008166547864675522, 0.023945670574903488, 0.08047983795404434, 0.04428931325674057, 0.048865146934986115, 0.23266322910785675, 0.38709723949432373, 0.04732646048069, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.003697684034705162, 0.00645096693187952, 0.000492173305246979, 0.0003069596423301846, 0.00012400784180499613, 0.0010727369226515293, 0.0008858798537403345, 0.0013806369388476014, 0.0008200189913623035, 0.00027311884332448244, 0.00278306077234447, 0.0352943018078804, 0.0035613805521279573, 0.007148319855332375, 0.0016474071890115738, 0.016731666401028633, 0.01917283609509468, 0.022339235991239548, 0.87336665391922, 0.0024509592913091183, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.013628801330924034, 0.0014384459936991334, 0.0012188645778223872, 0.0012915869010612369, 0.000611803203355521, 0.0021400409750640392, 0.002746956655755639, 0.0033360454253852367, 0.003091871039941907, 0.00598891731351614, 0.007237218786031008, 0.0035477408673614264, 0.012891970574855804, 0.022229138761758804, 0.02082422375679016, 0.026295306161046028, 0.0764816477894783, 0.16261456906795502, 0.1263866275548935, 0.10034438222646713, 0.40565383434295654, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.021432962268590927, 0.0007291652145795524, 0.0010889674304053187, 0.0008559622801840305, 0.00105566275306046, 0.002354246564209461, 0.0018633351428434253, 0.002486991696059704, 0.004011376760900021, 0.003645628225058317, 0.006636615376919508, 0.0024591649416834116, 0.008106639608740807, 0.01950140856206417, 0.013945518061518669, 0.016831574961543083, 0.06727857887744904, 0.10203741490840912, 0.07506183534860611, 0.03785502910614014, 0.34741589426994324, 0.2633461058139801, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018374430015683174, 0.001634014886803925, 0.00026486621936783195, 0.0006277103675529361, 0.0007212902419269085, 0.002294712932780385, 0.001437578583136201, 0.00243357359431684, 0.0010225047590211034, 0.0021730971056967974, 0.005665974225848913, 0.0005857078358530998, 0.004730312153697014, 0.01578964665532112, 0.009121054783463478, 0.006447888445109129, 0.04965188354253769, 0.07221101969480515, 0.04530423879623413, 0.02253623865544796, 0.23380506038665771, 0.28100937604904175, 0.22215783596038818, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016010284423828125, 0.0004190765612293035, 0.001679316977970302, 0.002033165656030178, 0.0011126059107482433, 0.002304123481735587, 0.001641487586311996, 0.0020452316384762526, 0.0008826431585475802, 0.003531817113980651, 0.0046607451513409615, 0.0011461281683295965, 0.004025280475616455, 0.011729471385478973, 0.006906012073159218, 0.005978975910693407, 0.03413613885641098, 0.05391993373632431, 0.07536928355693817, 0.015544145368039608, 0.1530727595090866, 0.2167249321937561, 0.2326735109090805, 0.15245290100574493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01641550101339817, 0.0010881011839956045, 0.00130270270165056, 0.0012335636420175433, 0.0010325246257707477, 0.0013605300337076187, 0.0011352745350450277, 0.000971320434473455, 0.001758022466674447, 0.0023549969773739576, 0.002267795614898205, 0.0026170075871050358, 0.0032849586568772793, 0.005564443301409483, 0.007287745364010334, 0.006582664791494608, 0.016911091282963753, 0.023348135873675346, 0.033363793045282364, 0.012060877867043018, 0.08410520851612091, 0.10776975750923157, 0.12741316854953766, 0.2500956654548645, 0.28867512941360474, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.031117701902985573, 0.0015063050668686628, 0.0010839310707524419, 0.0024215243756771088, 0.0013826877111569047, 0.003793720155954361, 0.001633927458897233, 0.0031195192132145166, 0.0012812637723982334, 0.002068526344373822, 0.004760111682116985, 0.0008757317555136979, 0.008592111989855766, 0.009353652596473694, 0.011993249878287315, 0.005963003262877464, 0.022028181701898575, 0.034113261848688126, 0.009586761705577374, 0.013681815937161446, 0.08417057245969772, 0.0891781747341156, 0.10806850343942642, 0.1556079387664795, 0.37236833572387695, 0.02024947665631771, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01483848039060831, 0.000575732090510428, 0.00474366033449769, 0.00045677367597818375, 0.000741229101549834, 0.0015000016428530216, 0.0008866065181791782, 0.0008452082402072847, 0.00010359263978898525, 0.0009065119666047394, 0.0018096778076142073, 0.0602911114692688, 0.0015377566451206803, 0.0033079409040510654, 0.0013713662046939135, 0.0020826742984354496, 0.008643998764455318, 0.009088877588510513, 0.007760961540043354, 0.00580832501873374, 0.03386510908603668, 0.039871808141469955, 0.0537455677986145, 0.036947716027498245, 0.1225944235920906, 0.5718703866004944, 0.013804464600980282, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018570026382803917, 0.001214316813275218, 0.001288045896217227, 0.0010612581390887499, 0.0007490055286325514, 0.0010485630482435226, 0.0008687878726050258, 0.000597386562731117, 0.0012938298750668764, 0.0014042515540495515, 0.0011055077193304896, 0.0005003451951779425, 0.0020262610632926226, 0.002146316459402442, 0.002873787423595786, 0.0026413691230118275, 0.00580298388376832, 0.00819135457277298, 0.008339088410139084, 0.00554330600425601, 0.02655870094895363, 0.033089183270931244, 0.032507363706827164, 0.09199850261211395, 0.10410479456186295, 0.16861777007579803, 0.20895445346832275, 0.2669033408164978, 0.0, 0.0, 0.0, 0.0, 0.0], [0.016654420644044876, 0.0007248660549521446, 0.0008980656857602298, 0.003161691827699542, 0.000552282202988863, 0.0016607262659817934, 0.0007486115209758282, 0.0009652800508774817, 0.00038428022526204586, 0.0022577778436243534, 0.001554560731165111, 0.0007036995957605541, 0.0014688166556879878, 0.0027998804580420256, 0.0030139742884784937, 0.00536904064938426, 0.00655234744772315, 0.008701564744114876, 0.0071388729847967625, 0.006516325753182173, 0.02618558146059513, 0.039291974157094955, 0.031550075858831406, 0.024343032389879227, 0.10670769214630127, 0.15159782767295837, 0.13674220442771912, 0.32013964653015137, 0.09161482006311417, 0.0, 0.0, 0.0, 0.0], [0.03722681477665901, 0.0007442743517458439, 0.0019180347444489598, 0.001429063268005848, 0.0025597354397177696, 0.003292833222076297, 0.000900725310202688, 0.0015445080352947116, 0.00035572092747315764, 0.0004492734733503312, 0.002668817061930895, 0.0014046675059944391, 0.0017280991887673736, 0.0045427363365888596, 0.0012941062450408936, 0.0046776654198765755, 0.009138201363384724, 0.00899294950067997, 0.005343651864677668, 0.002031840616837144, 0.030930932611227036, 0.021215267479419708, 0.03639978542923927, 0.056794650852680206, 0.11195951700210571, 0.011458112858235836, 0.03353293985128403, 0.26588523387908936, 0.24695412814617157, 0.0926256850361824, 0.0, 0.0, 0.0], [0.04408871755003929, 0.0011082934215664864, 0.003387592500075698, 0.00485319085419178, 0.002864320296794176, 0.004500855691730976, 0.0014518105890601873, 0.002192494459450245, 0.000790234946180135, 0.0006717384676449001, 0.003577150171622634, 0.0005131771322339773, 0.012861711904406548, 0.005116850603371859, 0.003014254616573453, 0.005887453909963369, 0.010453412309288979, 0.013818330131471157, 0.008691062219440937, 0.011229278519749641, 0.03140084818005562, 0.023181620985269547, 0.03044438175857067, 0.035206835716962814, 0.0999986007809639, 0.006090162321925163, 0.147142231464386, 0.1639304906129837, 0.12749530375003815, 0.1935776025056839, 0.00045999058056622744, 0.0, 0.0], [0.037174031138420105, 0.005006894469261169, 0.0016112328739836812, 0.0028301526326686144, 0.0005118482513353229, 0.0027071235235780478, 0.0020970292389392853, 0.0012328780721873045, 0.0006310881581157446, 0.0017632455565035343, 0.0017868787981569767, 0.0002528333861846477, 0.0010720735881477594, 0.0026448341086506844, 0.0015661584911867976, 0.003808410372585058, 0.0052767470479011536, 0.006930259056389332, 0.013310844078660011, 0.00358164613135159, 0.01830451376736164, 0.02668476104736328, 0.019971540197730064, 0.032155752182006836, 0.07432039827108383, 0.02392224594950676, 0.08761483430862427, 0.18007564544677734, 0.027492497116327286, 0.059722427278757095, 0.1912268102169037, 0.16271233558654785, 0.0], [0.0081169493496418, 0.0008723055943846703, 0.0003707291616592556, 0.0004734861431643367, 0.00017833011224865913, 0.0003994707949459553, 0.00029511729371733963, 0.00018311968597117811, 0.00016714615048840642, 0.00023702486942056566, 0.0002533361839596182, 0.00012855041131842881, 0.00020213212701492012, 0.0003994494618382305, 0.00016511819558218122, 0.0004622876294888556, 0.0008780236239545047, 0.0011140479473397136, 0.0020708832889795303, 0.0019636782817542553, 0.003527858527377248, 0.004475224297493696, 0.004463494755327702, 0.007312549278140068, 0.014446238987147808, 0.03503050282597542, 0.020014189183712006, 0.043885767459869385, 0.019436748698353767, 0.11325743794441223, 0.07277625054121017, 0.16196835041046143, 0.48047420382499695]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473748207092285, 0.05262522026896477, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8785883784294128, 0.055394671857357025, 0.06601690500974655, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7270770072937012, 0.13709871470928192, 0.04866093024611473, 0.08716334402561188, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.46517449617385864, 0.13429313898086548, 0.1535816639661789, 0.17472556233406067, 0.0722251832485199, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1703597903251648, 0.011841939762234688, 0.02365085668861866, 0.0217458326369524, 0.025856344029307365, 0.7465452551841736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14214572310447693, 0.04383271932601929, 0.05021180212497711, 0.045165061950683594, 0.058721303939819336, 0.5111533403396606, 0.1487700343132019, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0847334936261177, 0.020369820296764374, 0.03746162727475166, 0.03195912018418312, 0.034196022897958755, 0.4350067675113678, 0.11894061416387558, 0.2373325079679489, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.34230807423591614, 0.03593635931611061, 0.07259213179349899, 0.10352759808301926, 0.08196530491113663, 0.07694563269615173, 0.1419631987810135, 0.09896712005138397, 0.04579458758234978, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3711484968662262, 0.06278740614652634, 0.05806949734687805, 0.05566273629665375, 0.13124844431877136, 0.07535625994205475, 0.10726634413003922, 0.07924025505781174, 0.030107328668236732, 0.029113227501511574, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.06430106610059738, 0.00516910757869482, 0.011013001203536987, 0.009054888971149921, 0.011748075485229492, 0.31885015964508057, 0.0368494875729084, 0.08319544047117233, 0.012068144045770168, 0.014676232822239399, 0.43307432532310486, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23777925968170166, 0.03812996298074722, 0.01974337175488472, 0.0750802680850029, 0.14420796930789948, 0.06244926154613495, 0.07018382102251053, 0.06523606181144714, 0.04883221164345741, 0.034600358456373215, 0.06439772248268127, 0.13935977220535278, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29014483094215393, 0.04085767641663551, 0.06965622305870056, 0.06467738747596741, 0.11515596508979797, 0.0727943703532219, 0.0540604293346405, 0.04095996171236038, 0.03646121546626091, 0.037096962332725525, 0.08112158626317978, 0.06249444559216499, 0.034518927335739136, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04191604629158974, 0.0033880716655403376, 0.007082365918904543, 0.005899209063500166, 0.007586199790239334, 0.2038978785276413, 0.023296775296330452, 0.05173072591423988, 0.00781681016087532, 0.00973138865083456, 0.27760186791419983, 0.008386769331991673, 0.01617272011935711, 0.33549317717552185, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15129354596138, 0.042407918721437454, 0.04383376985788345, 0.09706765413284302, 0.14943547546863556, 0.04439881816506386, 0.06174508482217789, 0.04199513792991638, 0.018608784303069115, 0.031412433832883835, 0.050926435738801956, 0.06660289317369461, 0.0941622257232666, 0.05761288106441498, 0.04849691689014435, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2995803654193878, 0.07432068139314651, 0.030061939731240273, 0.016736488789319992, 0.03856480494141579, 0.051068760454654694, 0.09408658742904663, 0.05606793984770775, 0.05458230897784233, 0.04522039741277695, 0.05638067424297333, 0.015378037467598915, 0.04082850366830826, 0.06084688380360603, 0.030811060220003128, 0.03546453267335892, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03043997474014759, 0.002493392676115036, 0.005298209376633167, 0.004223390016704798, 0.005517452955245972, 0.14508455991744995, 0.016034148633480072, 0.03598649799823761, 0.005758052691817284, 0.007398176938295364, 0.19613459706306458, 0.006090890150517225, 0.011465122923254967, 0.2373799830675125, 0.00803177896887064, 0.005861485376954079, 0.27680227160453796, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01566893421113491, 0.0019133493769913912, 0.006747938226908445, 0.005610875319689512, 0.006710364017635584, 0.10858526825904846, 0.013436980545520782, 0.030674800276756287, 0.008698046207427979, 0.00595876993611455, 0.15696406364440918, 0.0064012641087174416, 0.014357205480337143, 0.19220466911792755, 0.00993881095200777, 0.008154558017849922, 0.22861401736736298, 0.1793600618839264, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2751207649707794, 0.046356093138456345, 0.025782747194170952, 0.020371075719594955, 0.045191291719675064, 0.04741503298282623, 0.0580083392560482, 0.05449455976486206, 0.02185583859682083, 0.0286741741001606, 0.05054797977209091, 0.03578946739435196, 0.014343999326229095, 0.055051881819963455, 0.019970901310443878, 0.025070667266845703, 0.05983675271272659, 0.10591266304254532, 0.010205714032053947, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12210462242364883, 0.029006371274590492, 0.02792610600590706, 0.030763698741793633, 0.06431958079338074, 0.050758663564920425, 0.057467177510261536, 0.038544513285160065, 0.02881445921957493, 0.036410387605428696, 0.05530690401792526, 0.02755567617714405, 0.05062384530901909, 0.060396790504455566, 0.03790905699133873, 0.03416259214282036, 0.06550323963165283, 0.08224610984325409, 0.08903133124113083, 0.011148842982947826, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.019922727718949318, 0.0015991271939128637, 0.003536272794008255, 0.0027606075163930655, 0.0036747511476278305, 0.09198573231697083, 0.009868195280432701, 0.021762719377875328, 0.0036746214609593153, 0.004800302442163229, 0.12327175587415695, 0.003948227036744356, 0.007727707736194134, 0.14901940524578094, 0.005094499792903662, 0.003849549451842904, 0.17433807253837585, 0.1450374871492386, 0.0031391791999340057, 0.003901566145941615, 0.21708737313747406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.025169137865304947, 0.0025891424156725407, 0.007904481142759323, 0.002818176755681634, 0.008317500352859497, 0.06284766644239426, 0.01609736867249012, 0.024833573028445244, 0.007658042479306459, 0.011317076161503792, 0.08451000601053238, 0.012135406956076622, 0.02204839326441288, 0.1001725122332573, 0.015103284269571304, 0.0201522558927536, 0.11906812340021133, 0.1414896845817566, 0.006576654501259327, 0.0070740594528615475, 0.14850661158561707, 0.15361088514328003, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08679646253585815, 0.01581869274377823, 0.009913157671689987, 0.005941125098615885, 0.011749111115932465, 0.04402248561382294, 0.049411579966545105, 0.047211579978466034, 0.022448495030403137, 0.028144152835011482, 0.05249372124671936, 0.013664900325238705, 0.038044560700654984, 0.058564912527799606, 0.016008716076612473, 0.028531773015856743, 0.0675240010023117, 0.1000405102968216, 0.02607448399066925, 0.018165268003940582, 0.08061444759368896, 0.14588388800621033, 0.03293200582265854, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12203321605920792, 0.026796909049153328, 0.03471270203590393, 0.016600852832198143, 0.050985999405384064, 0.03362344577908516, 0.03376288339495659, 0.03289441019296646, 0.022763598710298538, 0.03469546511769295, 0.03710825741291046, 0.02750304900109768, 0.030508041381835938, 0.04070490226149559, 0.019409803673624992, 0.03338814154267311, 0.04603789001703262, 0.08141937106847763, 0.025792937725782394, 0.01771334744989872, 0.053263794630765915, 0.06955219805240631, 0.07340095192193985, 0.03532779961824417, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.01263270154595375, 0.00319672585465014, 0.008999624289572239, 0.0037871042732149363, 0.005590219981968403, 0.0601874515414238, 0.010173020884394646, 0.02606339193880558, 0.00828032847493887, 0.005998439155519009, 0.08480240404605865, 0.007755734026432037, 0.014807655476033688, 0.10079342871904373, 0.011636023409664631, 0.007885100319981575, 0.12061366438865662, 0.12819865345954895, 0.006487114820629358, 0.0032305698841810226, 0.15047509968280792, 0.10557690262794495, 0.015480606816709042, 0.01902632601559162, 0.0783216655254364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11176055669784546, 0.018635569140315056, 0.05622375011444092, 0.046772830188274384, 0.03736492618918419, 0.022993402555584908, 0.0313652902841568, 0.026408856734633446, 0.04335922375321388, 0.056635573506355286, 0.021786397323012352, 0.023822134360671043, 0.03064168244600296, 0.023289253935217857, 0.05430489033460617, 0.06881187111139297, 0.024489376693964005, 0.047770872712135315, 0.03627173975110054, 0.022222528234124184, 0.02699895016849041, 0.02176610380411148, 0.03442489728331566, 0.028240276500582695, 0.05891920626163483, 0.02471984177827835, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10639800876379013, 0.037107907235622406, 0.08273960649967194, 0.01785687543451786, 0.020950013771653175, 0.018305251374840736, 0.022620096802711487, 0.018192263320088387, 0.021851176396012306, 0.012518547475337982, 0.019214089959859848, 0.030340079218149185, 0.07187619060277939, 0.020788004621863365, 0.03963523358106613, 0.06151588261127472, 0.023179279640316963, 0.04398038983345032, 0.04619225114583969, 0.020667986944317818, 0.02708088420331478, 0.04386553168296814, 0.03276738524436951, 0.04369066283106804, 0.050808411091566086, 0.04029491916298866, 0.025563009083271027, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.015084551647305489, 0.003822381142526865, 0.005877269431948662, 0.0049718525260686874, 0.005370154045522213, 0.052236441522836685, 0.016127681359648705, 0.03182988241314888, 0.008211125619709492, 0.011473821476101875, 0.06704682856798172, 0.010528388433158398, 0.010598231106996536, 0.07938896119594574, 0.007211719639599323, 0.006271461024880409, 0.09142373502254486, 0.10866507142782211, 0.003973817452788353, 0.005621669348329306, 0.10983327031135559, 0.09237755835056305, 0.022960485890507698, 0.01931491121649742, 0.0886414498090744, 0.0184647087007761, 0.011022255755960941, 0.0916503369808197, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09854129701852798, 0.03967062011361122, 0.05052143707871437, 0.04794389754533768, 0.01953187584877014, 0.015683410689234734, 0.025074753910303116, 0.014997918158769608, 0.027723444625735283, 0.016418198123574257, 0.0163890291005373, 0.026316557079553604, 0.034359268844127655, 0.01775798574090004, 0.023362653329968452, 0.037635158747434616, 0.019405605271458626, 0.034421682357788086, 0.049022965133190155, 0.030190536752343178, 0.022159066051244736, 0.03754064813256264, 0.038793135434389114, 0.04032411426305771, 0.0423126295208931, 0.06829200685024261, 0.02531411126255989, 0.04330069199204445, 0.03699525445699692, 0.0, 0.0, 0.0, 0.0], [0.10688815265893936, 0.03659575805068016, 0.005083092488348484, 0.010160422883927822, 0.013297378085553646, 0.041752077639102936, 0.026246795430779457, 0.03012855537235737, 0.013482393696904182, 0.01822320930659771, 0.04582623392343521, 0.02555878274142742, 0.015618828125298023, 0.050398919731378555, 0.011657006107270718, 0.011659118346869946, 0.055155202746391296, 0.05091662332415581, 0.019206788390874863, 0.013998415321111679, 0.06383471190929413, 0.02908891998231411, 0.016710437834262848, 0.026095272973179817, 0.054849836975336075, 0.038066744804382324, 0.03896068409085274, 0.05375230684876442, 0.023679163306951523, 0.05310814082622528, 0.0, 0.0, 0.0], [0.10871606320142746, 0.06286418437957764, 0.02304510585963726, 0.01249321736395359, 0.009688264690339565, 0.023720761761069298, 0.02561734989285469, 0.028282862156629562, 0.019523102790117264, 0.015187134966254234, 0.024615760892629623, 0.01885579712688923, 0.013071930967271328, 0.025697439908981323, 0.010639766231179237, 0.007336437702178955, 0.0284882839769125, 0.04087512195110321, 0.023693948984146118, 0.028460824862122536, 0.031781166791915894, 0.03399212658405304, 0.01690450869500637, 0.04699752479791641, 0.054809194058179855, 0.060481712222099304, 0.021137820556759834, 0.05147768557071686, 0.015304340049624443, 0.07868866622447968, 0.03755193576216698, 0.0, 0.0], [0.16510622203350067, 0.013307326473295689, 0.022820325568318367, 0.03222498670220375, 0.028327837586402893, 0.011912121437489986, 0.02175913192331791, 0.01259133405983448, 0.025371316820383072, 0.015923479571938515, 0.011745063588023186, 0.021340154111385345, 0.022658515721559525, 0.0120970094576478, 0.05877388268709183, 0.041142966598272324, 0.01286615151911974, 0.021896688267588615, 0.02738574519753456, 0.015412732027471066, 0.01482226513326168, 0.01319878175854683, 0.029060123488307, 0.04031302407383919, 0.023036932572722435, 0.05035791173577309, 0.04650065675377846, 0.032210756093263626, 0.045741356909275055, 0.01144817378371954, 0.051484838128089905, 0.04716221243143082, 0.0], [0.019862735643982887, 0.0006941372412256896, 0.0017670637462288141, 0.001903822529129684, 0.0023115796502679586, 0.05924385413527489, 0.005164946895092726, 0.00871454831212759, 0.0024906715843826532, 0.0030579622834920883, 0.07585153728723526, 0.002390553243458271, 0.0031773524824529886, 0.08945117145776749, 0.003065295284613967, 0.001620917348191142, 0.10312028974294662, 0.06259157508611679, 0.002058297162875533, 0.002232220023870468, 0.12742048501968384, 0.024703865870833397, 0.003510191338136792, 0.004994176793843508, 0.02831714041531086, 0.00587420491501689, 0.006215326488018036, 0.03686561807990074, 0.00543811172246933, 0.004061863757669926, 0.004735524300485849, 0.004534377250820398, 0.2925585210323334]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9726406335830688, 0.02735934779047966, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8850567936897278, 0.081606425344944, 0.03333687782287598, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7844979166984558, 0.09045999497175217, 0.10720972716808319, 0.01783234439790249, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6679065823554993, 0.10135091841220856, 0.10997925698757172, 0.0699087604880333, 0.050854481756687164, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.586279034614563, 0.06960930675268173, 0.06798025965690613, 0.06058964133262634, 0.08339769393205643, 0.13214407861232758, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5015711188316345, 0.062278736382722855, 0.07209991663694382, 0.055909160524606705, 0.08146880567073822, 0.12033165246248245, 0.10634063929319382, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4001610577106476, 0.05525398254394531, 0.0634768009185791, 0.054137859493494034, 0.07119619846343994, 0.11230526864528656, 0.12489724904298782, 0.11857163161039352, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4018532931804657, 0.08579416573047638, 0.05644882842898369, 0.0718112513422966, 0.08318912237882614, 0.07845644652843475, 0.09191881865262985, 0.07768875360488892, 0.052839308977127075, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3827159106731415, 0.07976946234703064, 0.06207754462957382, 0.06360232830047607, 0.08211029320955276, 0.07804952561855316, 0.08597437292337418, 0.07506486773490906, 0.0632784441113472, 0.027357211336493492, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3404167592525482, 0.04535197839140892, 0.04658910259604454, 0.0403917022049427, 0.05489041656255722, 0.08179005235433578, 0.08906517177820206, 0.09645894169807434, 0.055649686604738235, 0.06398673355579376, 0.08540939539670944, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3118618130683899, 0.09539999067783356, 0.07292018085718155, 0.06976873427629471, 0.07777831703424454, 0.06735038757324219, 0.061614058911800385, 0.05481072887778282, 0.052520155906677246, 0.050225190818309784, 0.07299350202083588, 0.01275695487856865, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3665285110473633, 0.06308675557374954, 0.03947846218943596, 0.046095237135887146, 0.057407598942518234, 0.06348336488008499, 0.052384644746780396, 0.055016204714775085, 0.04584016650915146, 0.06228691339492798, 0.06589911133050919, 0.058063603937625885, 0.02442951127886772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2856442928314209, 0.03804026544094086, 0.039423901587724686, 0.03423863276839256, 0.04639400169253349, 0.06771663576364517, 0.07328075170516968, 0.07855430990457535, 0.04789666086435318, 0.0559365451335907, 0.07124951481819153, 0.035689257085323334, 0.049560435116291046, 0.07637479901313782, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2977542579174042, 0.052595481276512146, 0.043479401618242264, 0.0584585964679718, 0.04568730294704437, 0.052594054490327835, 0.044561997056007385, 0.04597475007176399, 0.06316148489713669, 0.07768756151199341, 0.054991479963064194, 0.03466470167040825, 0.05020098388195038, 0.0588613823056221, 0.019326532259583473, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.28853148221969604, 0.04616909474134445, 0.028233075514435768, 0.03824242204427719, 0.043306268751621246, 0.051749151200056076, 0.04318273440003395, 0.04393118619918823, 0.05616343393921852, 0.06773816794157028, 0.054901786148548126, 0.08139313757419586, 0.04325690120458603, 0.059048131108284, 0.03989902883768082, 0.014254056848585606, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.23721377551555634, 0.03161582350730896, 0.03324565663933754, 0.029231935739517212, 0.03861813619732857, 0.055491674691438675, 0.05966497212648392, 0.06321780383586884, 0.040737003087997437, 0.047526754438877106, 0.05837823823094368, 0.030399667099118233, 0.042871564626693726, 0.06284615397453308, 0.05321288853883743, 0.04736895114183426, 0.06835902482271194, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.20986966788768768, 0.027291255071759224, 0.031156612560153008, 0.026615014299750328, 0.03165114298462868, 0.05273226276040077, 0.0527450293302536, 0.056359030306339264, 0.04076547548174858, 0.04278237745165825, 0.05619126558303833, 0.03067827597260475, 0.04398961365222931, 0.060914747416973114, 0.050136975944042206, 0.04157315939664841, 0.06652863323688507, 0.07801954448223114, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2385382056236267, 0.044145818799734116, 0.04643506556749344, 0.0345127135515213, 0.03572182357311249, 0.04369329661130905, 0.04304974526166916, 0.034988246858119965, 0.04371808469295502, 0.0395219549536705, 0.04457855597138405, 0.02459944784641266, 0.0625118538737297, 0.04753481224179268, 0.04375608637928963, 0.04995930939912796, 0.05090807005763054, 0.0526624396443367, 0.019164465367794037, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17729148268699646, 0.04221462085843086, 0.030818719416856766, 0.03088611178100109, 0.03386850282549858, 0.045211877673864365, 0.04290103167295456, 0.03221558406949043, 0.040188632905483246, 0.04926898330450058, 0.0480162538588047, 0.038142431527376175, 0.05126511678099632, 0.052068427205085754, 0.04744118079543114, 0.03924531117081642, 0.05630476027727127, 0.05462942644953728, 0.0599186047911644, 0.028102951124310493, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19048278033733368, 0.025203481316566467, 0.02678745612502098, 0.023672685027122498, 0.031104043126106262, 0.04345342889428139, 0.04649331793189049, 0.048206474632024765, 0.03323278948664665, 0.03892752528190613, 0.04549149051308632, 0.024442732334136963, 0.03570157289505005, 0.04907767102122307, 0.04303562641143799, 0.03905563801527023, 0.0537174716591835, 0.06658954918384552, 0.043208248913288116, 0.03176471218466759, 0.06035126745700836, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14975358545780182, 0.023474570363759995, 0.025508398190140724, 0.02894243411719799, 0.03517160564661026, 0.038801901042461395, 0.04594281315803528, 0.03962729126214981, 0.03708778694272041, 0.03872300684452057, 0.04190720245242119, 0.021688057109713554, 0.04328255355358124, 0.04592541977763176, 0.03893951326608658, 0.04007609561085701, 0.050600212067365646, 0.060830507427453995, 0.04372159391641617, 0.03023524582386017, 0.057286132127046585, 0.062473997473716736, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1751340925693512, 0.027519281953573227, 0.029514020308852196, 0.020715320482850075, 0.026873527094721794, 0.0345461405813694, 0.03853079304099083, 0.03303780034184456, 0.041815582662820816, 0.040100324898958206, 0.03696896508336067, 0.030468929558992386, 0.03995395079255104, 0.04023703560233116, 0.04482618346810341, 0.03708260878920555, 0.044201456010341644, 0.04931040480732918, 0.04125744849443436, 0.02861069142818451, 0.04997344687581062, 0.05269284173846245, 0.036629173904657364, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13285991549491882, 0.04070806875824928, 0.04111679270863533, 0.03484426438808441, 0.04087606817483902, 0.02756720781326294, 0.028293685987591743, 0.022520428523421288, 0.0387541800737381, 0.04037804901599884, 0.029089422896504402, 0.02960922010242939, 0.05670115351676941, 0.031181858852505684, 0.037661172449588776, 0.056886348873376846, 0.03396708145737648, 0.037813056260347366, 0.04951411858201027, 0.03194557875394821, 0.037599027156829834, 0.043266359716653824, 0.04305668920278549, 0.033790234476327896, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14709366858005524, 0.018218714743852615, 0.02214360423386097, 0.02162138931453228, 0.027072900906205177, 0.0338541679084301, 0.037876129150390625, 0.03801964595913887, 0.02793251909315586, 0.030901839956641197, 0.036069247871637344, 0.01902312971651554, 0.03465802222490311, 0.039169859141111374, 0.0337555818259716, 0.026136614382267, 0.04297831654548645, 0.05313740670681, 0.03463505581021309, 0.02014029584825039, 0.048783592879772186, 0.049908023327589035, 0.046243924647569656, 0.0471566878259182, 0.06346960365772247, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.17602786421775818, 0.014882153831422329, 0.02675117552280426, 0.02408803254365921, 0.035667501389980316, 0.027094297111034393, 0.024984251707792282, 0.02147187665104866, 0.026359377428889275, 0.0238451287150383, 0.027566984295845032, 0.01686953753232956, 0.0777743011713028, 0.029109222814440727, 0.04799697920680046, 0.029002491384744644, 0.03132336214184761, 0.03615597262978554, 0.03959561139345169, 0.017008012160658836, 0.03483906015753746, 0.03443717956542969, 0.041492193937301636, 0.09124378114938736, 0.0405229777097702, 0.003890601685270667, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1850174069404602, 0.027787018567323685, 0.025097638368606567, 0.027018997818231583, 0.029199223965406418, 0.030337018892169, 0.032221779227256775, 0.02301507443189621, 0.026792876422405243, 0.0227938462048769, 0.030674856156110764, 0.021220847964286804, 0.053888071328401566, 0.0325930118560791, 0.03073764033615589, 0.025725238025188446, 0.03486606106162071, 0.037935808300971985, 0.02867201343178749, 0.026148974895477295, 0.038753341883420944, 0.041795019060373306, 0.03863157331943512, 0.05337151885032654, 0.039820749312639236, 0.02571161836385727, 0.010172740556299686, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.12446323782205582, 0.01676890440285206, 0.020105106756091118, 0.019267026335000992, 0.024457838386297226, 0.03020336851477623, 0.031273938715457916, 0.032546062022447586, 0.023186005651950836, 0.02350839599967003, 0.03173321112990379, 0.0170805174857378, 0.02908204309642315, 0.03428217023611069, 0.03579128161072731, 0.025820527225732803, 0.037531182169914246, 0.04618248715996742, 0.029500000178813934, 0.017913812771439552, 0.04274730384349823, 0.0424346886575222, 0.039002396166324615, 0.04141836240887642, 0.059495024383068085, 0.035957351326942444, 0.030984869226813316, 0.0572628453373909, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14491020143032074, 0.025843990966677666, 0.0311769787222147, 0.01946275122463703, 0.012575741857290268, 0.02527458593249321, 0.022020690143108368, 0.021267294883728027, 0.03165541961789131, 0.027664169669151306, 0.026251602917909622, 0.02209104411303997, 0.038517940789461136, 0.028266875073313713, 0.02887759543955326, 0.036865346133708954, 0.030699962750077248, 0.032045040279626846, 0.03411627560853958, 0.02035541459918022, 0.03446316719055176, 0.03542863577604294, 0.0307284127920866, 0.04441893845796585, 0.03793597221374512, 0.057348668575286865, 0.03889700025320053, 0.04111207276582718, 0.019728200510144234, 0.0, 0.0, 0.0, 0.0], [0.13168425858020782, 0.025460127741098404, 0.02796241268515587, 0.02410983480513096, 0.016868818551301956, 0.03140678629279137, 0.02324734628200531, 0.02207169122993946, 0.0273625198751688, 0.026370463892817497, 0.03204534575343132, 0.028775809332728386, 0.046142008155584335, 0.034559763967990875, 0.03132105991244316, 0.02943774126470089, 0.037490688264369965, 0.03597423434257507, 0.0306710135191679, 0.026778336614370346, 0.04166237264871597, 0.03274395689368248, 0.03034876473248005, 0.03265197202563286, 0.038056496530771255, 0.03457563742995262, 0.0229361429810524, 0.0369032584130764, 0.025966189801692963, 0.014414897188544273, 0.0, 0.0, 0.0], [0.1656370609998703, 0.022809254005551338, 0.02910775877535343, 0.024944672361016273, 0.02435288392007351, 0.02503182739019394, 0.018486671149730682, 0.01857367716729641, 0.02844363823533058, 0.029199928045272827, 0.02531822957098484, 0.03220498189330101, 0.025753289461135864, 0.026890520006418228, 0.031273942440748215, 0.0330515056848526, 0.028835458680987358, 0.029910435900092125, 0.027558818459510803, 0.024624671787023544, 0.032053619623184204, 0.026493750512599945, 0.02170960046350956, 0.03164014592766762, 0.03707476332783699, 0.047994691878557205, 0.02793215773999691, 0.034756116569042206, 0.03804944083094597, 0.02153567597270012, 0.008750859647989273, 0.0, 0.0], [0.16794218122959137, 0.025983458384871483, 0.03222336247563362, 0.019469374790787697, 0.023735281080007553, 0.023115724325180054, 0.019973624497652054, 0.016550222411751747, 0.029796773567795753, 0.02849467657506466, 0.023108838126063347, 0.02208707481622696, 0.04216558113694191, 0.02439017780125141, 0.02074914425611496, 0.038735855370759964, 0.026269804686307907, 0.027289103716611862, 0.019617382436990738, 0.020242871716618538, 0.029203206300735474, 0.03041333518922329, 0.02576262876391411, 0.04084981232881546, 0.033326659351587296, 0.021795304492115974, 0.026310529559850693, 0.03101350925862789, 0.0368926078081131, 0.019377173855900764, 0.04355606436729431, 0.009558649733662605, 0.0], [0.11396826803684235, 0.01775287836790085, 0.01606195606291294, 0.015585051849484444, 0.018396127969026566, 0.024367081001400948, 0.026053886860609055, 0.023999111726880074, 0.021919485181570053, 0.026251107454299927, 0.02473055198788643, 0.015323014929890633, 0.02561456337571144, 0.0263378769159317, 0.024982862174510956, 0.02464262582361698, 0.02859032154083252, 0.03295150771737099, 0.02756861224770546, 0.018487561494112015, 0.032208509743213654, 0.03191468492150307, 0.03630725294351578, 0.0341540202498436, 0.03678558021783829, 0.03297772258520126, 0.026278916746377945, 0.03809890151023865, 0.029801947996020317, 0.018183259293437004, 0.03632069006562233, 0.03568092733621597, 0.057703081518411636]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8058414459228516, 0.19415852427482605, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.6373866200447083, 0.09672041982412338, 0.265892893075943, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5870266556739807, 0.09922719746828079, 0.07913825660943985, 0.23460792005062103, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.41947388648986816, 0.06312593072652817, 0.08123064041137695, 0.08381502330303192, 0.3523544669151306, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5308532118797302, 0.09590727835893631, 0.07779338210821152, 0.06661549210548401, 0.07293294370174408, 0.15589766204357147, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3850308358669281, 0.0747809112071991, 0.06978148967027664, 0.04760652780532837, 0.051449164748191833, 0.08776704967021942, 0.2835840880870819, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29629576206207275, 0.053282689303159714, 0.06944747269153595, 0.04097547382116318, 0.06660830974578857, 0.1021856889128685, 0.10869500786066055, 0.26250964403152466, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3047499358654022, 0.06337984651327133, 0.057534221559762955, 0.040577203035354614, 0.0730014219880104, 0.07375828176736832, 0.07239890098571777, 0.05693603679537773, 0.25766417384147644, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.29880765080451965, 0.055528249591588974, 0.05443596467375755, 0.052853021770715714, 0.06568751484155655, 0.07554223388433456, 0.06968103349208832, 0.05411386862397194, 0.09120791405439377, 0.18214257061481476, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.26933014392852783, 0.06352430582046509, 0.059521228075027466, 0.05242471396923065, 0.06052158772945404, 0.12185833603143692, 0.07608655840158463, 0.09753073006868362, 0.04232607036828995, 0.05058327689766884, 0.10629312694072723, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2207430750131607, 0.06960074603557587, 0.048723481595516205, 0.06675716489553452, 0.06954209506511688, 0.06483786553144455, 0.04652373492717743, 0.0393008328974247, 0.018016086891293526, 0.03580629825592041, 0.053022872656583786, 0.2671257555484772, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1508007049560547, 0.04732472822070122, 0.12551118433475494, 0.062400851398706436, 0.05354631692171097, 0.0557839497923851, 0.04542611166834831, 0.03888601064682007, 0.03019980899989605, 0.033370550721883774, 0.04645954817533493, 0.023437917232513428, 0.28685227036476135, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2044883370399475, 0.04972049593925476, 0.048385463654994965, 0.04385359212756157, 0.051157187670469284, 0.1007489413022995, 0.06420683860778809, 0.08302562683820724, 0.03880453109741211, 0.04795774072408676, 0.09582383185625076, 0.03239699453115463, 0.04441105201840401, 0.0950193852186203, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1623673439025879, 0.028519166633486748, 0.04784306883811951, 0.04135480895638466, 0.040284883230924606, 0.0520053505897522, 0.04226506128907204, 0.038211677223443985, 0.05986882373690605, 0.06849399209022522, 0.047342803329229355, 0.019715161994099617, 0.06202227622270584, 0.045721933245658875, 0.24398359656333923, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13128934800624847, 0.03962808847427368, 0.059519387781620026, 0.03966321796178818, 0.044063545763492584, 0.04348495230078697, 0.037265460938215256, 0.03240035101771355, 0.037205737084150314, 0.06481598317623138, 0.040474534034729004, 0.025914225727319717, 0.06507597118616104, 0.03922510892152786, 0.07744196057319641, 0.22253204882144928, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1581333726644516, 0.03911251574754715, 0.03856496140360832, 0.03587968274950981, 0.04187794029712677, 0.08161000162363052, 0.05245806276798248, 0.0681268572807312, 0.034448061138391495, 0.043427519500255585, 0.08275996893644333, 0.029333388432860374, 0.041573796421289444, 0.08478687703609467, 0.048815999180078506, 0.032352857291698456, 0.08673813194036484, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13890594244003296, 0.031696319580078125, 0.03251555562019348, 0.035388994961977005, 0.038000330328941345, 0.06974424421787262, 0.05942453816533089, 0.06768238544464111, 0.027843810617923737, 0.039054661989212036, 0.07273104786872864, 0.030109861865639687, 0.041630931198596954, 0.07565011829137802, 0.04018831253051758, 0.02748361974954605, 0.0779590755701065, 0.09399021416902542, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09691290557384491, 0.030687307938933372, 0.03081795573234558, 0.044734325259923935, 0.04171943664550781, 0.036036763340234756, 0.04101704806089401, 0.032104771584272385, 0.04344063252210617, 0.04885552451014519, 0.03457902371883392, 0.01602509431540966, 0.049941662698984146, 0.0346352756023407, 0.06065687537193298, 0.03497787564992905, 0.03454343229532242, 0.03782793879508972, 0.2504861652851105, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1205921471118927, 0.04121064022183418, 0.050985176116228104, 0.05289152264595032, 0.02886129356920719, 0.04706200584769249, 0.0292128287255764, 0.03817000612616539, 0.021046485751867294, 0.029787806794047356, 0.044067203998565674, 0.026503337547183037, 0.04127003625035286, 0.044307369738817215, 0.0308015588670969, 0.04064902290701866, 0.044745102524757385, 0.042122453451156616, 0.028442276641726494, 0.1972716748714447, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11388302594423294, 0.028255384415388107, 0.028351284563541412, 0.026958664879202843, 0.03131086751818657, 0.05977547913789749, 0.0391821451485157, 0.050720393657684326, 0.027684016153216362, 0.03556424379348755, 0.06452134251594543, 0.024267084896564484, 0.03616494685411453, 0.06838168203830719, 0.04139300063252449, 0.029094647616147995, 0.0723004937171936, 0.07349381595849991, 0.037000201642513275, 0.0352863073348999, 0.07641096413135529, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08453850448131561, 0.01741660200059414, 0.018949145451188087, 0.016560642048716545, 0.0222961213439703, 0.041740600019693375, 0.058948639780282974, 0.038933612406253815, 0.030214646831154823, 0.03464233875274658, 0.0483190082013607, 0.016358420252799988, 0.025674551725387573, 0.05309673771262169, 0.02054652012884617, 0.02046983316540718, 0.05704639479517937, 0.06604909896850586, 0.03774643689393997, 0.02141387201845646, 0.06129930168390274, 0.2077389359474182, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11248438060283661, 0.024394843727350235, 0.024523146450519562, 0.027489561587572098, 0.026104668155312538, 0.03731505200266838, 0.06748531013727188, 0.03890374302864075, 0.01798143796622753, 0.028454391285777092, 0.03756532445549965, 0.01832607015967369, 0.02008865587413311, 0.038651708513498306, 0.023285649716854095, 0.02422662451863289, 0.03960611671209335, 0.048331886529922485, 0.045000918209552765, 0.017362141981720924, 0.04009140282869339, 0.05208287015557289, 0.19024406373500824, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08183358609676361, 0.019828448072075844, 0.026873398572206497, 0.029252560809254646, 0.025184448808431625, 0.03192773088812828, 0.02904980443418026, 0.024972524493932724, 0.03350626677274704, 0.04007430002093315, 0.033517323434352875, 0.016035988926887512, 0.03204454481601715, 0.03448064625263214, 0.028517091646790504, 0.03532731160521507, 0.03583228215575218, 0.04058396816253662, 0.029941104352474213, 0.02109791524708271, 0.036713290959596634, 0.03921259194612503, 0.03543240576982498, 0.23876048624515533, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10145686566829681, 0.014237044379115105, 0.01698417216539383, 0.018636561930179596, 0.028536243364214897, 0.036538016051054, 0.029570505023002625, 0.041983190923929214, 0.014166048727929592, 0.030081408098340034, 0.040166400372982025, 0.020075026899576187, 0.03037012182176113, 0.043534401804208755, 0.030393017455935478, 0.020717347040772438, 0.046771109104156494, 0.0549510233104229, 0.028809169307351112, 0.02004367858171463, 0.050925444811582565, 0.04084467887878418, 0.04185803234577179, 0.03848978132009506, 0.15986064076423645, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07937928289175034, 0.020674314349889755, 0.020799756050109863, 0.027988243848085403, 0.04565476253628731, 0.026562592014670372, 0.025396710261702538, 0.02124325931072235, 0.017763787880539894, 0.02520064078271389, 0.027576467022299767, 0.03060709312558174, 0.034380003809928894, 0.028568308800458908, 0.02272692695260048, 0.01605195738375187, 0.029712744057178497, 0.032761797308921814, 0.025702396407723427, 0.012878886424005032, 0.03062613122165203, 0.020369654521346092, 0.037413906306028366, 0.02785036712884903, 0.035718511790037155, 0.27639156579971313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.09605436027050018, 0.023092888295650482, 0.024199653416872025, 0.024974141269922256, 0.0368296317756176, 0.030014390125870705, 0.02106921747326851, 0.018117519095540047, 0.022569263353943825, 0.02834322862327099, 0.030505139380693436, 0.021707670763134956, 0.02821178175508976, 0.03163941949605942, 0.03916611522436142, 0.03627365827560425, 0.03311571851372719, 0.03514395281672478, 0.02574523724615574, 0.015889011323451996, 0.034675464034080505, 0.026770081371068954, 0.0271434485912323, 0.0404508039355278, 0.03760384023189545, 0.03232336789369583, 0.17837096750736237, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.07476936280727386, 0.011021513491868973, 0.010706555098295212, 0.0153254559263587, 0.01990356668829918, 0.03285866230726242, 0.021936064586043358, 0.029454141855239868, 0.016421504318714142, 0.022476589307188988, 0.03722343221306801, 0.017580224201083183, 0.021954141557216644, 0.04046214371919632, 0.031415414065122604, 0.010417777113616467, 0.044297389686107635, 0.05118126794695854, 0.029565544798970222, 0.01539432443678379, 0.049064263701438904, 0.032021939754486084, 0.030905121937394142, 0.041320525109767914, 0.06578274816274643, 0.039784032851457596, 0.02049969881772995, 0.1662566065788269, 0.0, 0.0, 0.0, 0.0, 0.0], [0.051242053508758545, 0.011390021070837975, 0.02026580087840557, 0.02452964149415493, 0.12732437252998352, 0.020083904266357422, 0.017692741006612778, 0.020381998270750046, 0.018548615276813507, 0.023206060752272606, 0.021026063710451126, 0.02052123285830021, 0.025210730731487274, 0.021994244307279587, 0.017190463840961456, 0.02080591581761837, 0.023035310208797455, 0.026458043605089188, 0.03108612634241581, 0.011454463936388493, 0.0243579912930727, 0.02252495475113392, 0.02460077963769436, 0.024588001891970634, 0.02838081680238247, 0.04660613089799881, 0.04362707585096359, 0.020300468429923058, 0.21156595647335052, 0.0, 0.0, 0.0, 0.0], [0.07929293066263199, 0.023537389934062958, 0.022847289219498634, 0.019936034455895424, 0.015462873503565788, 0.025110283866524696, 0.021429045125842094, 0.01507349219173193, 0.00992409698665142, 0.015724752098321915, 0.027188703417778015, 0.020747283473610878, 0.03497634828090668, 0.029102325439453125, 0.021219024434685707, 0.01814761571586132, 0.031058194115757942, 0.03380948677659035, 0.02013755589723587, 0.012323122471570969, 0.0336926244199276, 0.02433612011373043, 0.031084522604942322, 0.03377700597047806, 0.04355441406369209, 0.03296351432800293, 0.033149298280477524, 0.02536490187048912, 0.02216367982327938, 0.222866028547287, 0.0, 0.0, 0.0], [0.10059653222560883, 0.030963752418756485, 0.025578495115041733, 0.02022406831383705, 0.02713298238813877, 0.029490750283002853, 0.04074482619762421, 0.021403614431619644, 0.016392603516578674, 0.022520696744322777, 0.027555173262953758, 0.013599205762147903, 0.027306705713272095, 0.02780657820403576, 0.011214175261557102, 0.023586541414260864, 0.028399180620908737, 0.03066890686750412, 0.021664196625351906, 0.017032314091920853, 0.029051588848233223, 0.02654992789030075, 0.03297554329037666, 0.026790320873260498, 0.0320606492459774, 0.0272983405739069, 0.017307184636592865, 0.01876882277429104, 0.02822653204202652, 0.017085112631320953, 0.18000461161136627, 0.0, 0.0], [0.08361605554819107, 0.016581881791353226, 0.015823300927877426, 0.02200515754520893, 0.02406254969537258, 0.021028298884630203, 0.022811291739344597, 0.01441238820552826, 0.015572086907923222, 0.015104828402400017, 0.020917730405926704, 0.0217981468886137, 0.02598598599433899, 0.022013207897543907, 0.022025061771273613, 0.018091216683387756, 0.023115107789635658, 0.025451797991991043, 0.025943517684936523, 0.011241884902119637, 0.024466177448630333, 0.02286989800632, 0.03161345049738884, 0.029542675241827965, 0.02381790429353714, 0.029349012300372124, 0.025922631844878197, 0.017268184572458267, 0.0318446010351181, 0.01203774195164442, 0.0275249145925045, 0.25614133477211, 0.0], [0.05258439481258392, 0.012508377432823181, 0.012737678363919258, 0.012144195847213268, 0.014467820525169373, 0.024812445044517517, 0.01710914447903633, 0.02516256459057331, 0.016298964619636536, 0.0151600893586874, 0.0298200324177742, 0.014290275052189827, 0.024094799533486366, 0.0337427593767643, 0.021632259711623192, 0.018924115225672722, 0.03819216415286064, 0.04115181043744087, 0.024165775626897812, 0.020635640248656273, 0.0441853329539299, 0.037937093526124954, 0.029636353254318237, 0.031528398394584656, 0.045423079282045364, 0.03596361353993416, 0.03048253059387207, 0.04609917476773262, 0.03206765651702881, 0.01766294054687023, 0.03169601038098335, 0.044629115611314774, 0.10305343568325043]], [[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.770771324634552, 0.22922863066196442, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.7813623547554016, 0.10978527367115021, 0.1088523268699646, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5350388884544373, 0.11194939911365509, 0.15013684332370758, 0.2028748244047165, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4180358052253723, 0.1603638231754303, 0.10740304738283157, 0.1062578409910202, 0.20793947577476501, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.4349867105484009, 0.09579852223396301, 0.08251798897981644, 0.10028506070375443, 0.15332910418510437, 0.13308262825012207, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.44621366262435913, 0.056242119520902634, 0.05040298402309418, 0.0708429366350174, 0.09360908716917038, 0.1147262305021286, 0.16796302795410156, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.3609194755554199, 0.07294465601444244, 0.05191005393862724, 0.07159047573804855, 0.0932166576385498, 0.09543666988611221, 0.14539919793605804, 0.10858278721570969, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.31714069843292236, 0.07683125883340836, 0.0555228628218174, 0.06445365399122238, 0.08586593717336655, 0.10643252730369568, 0.1452113389968872, 0.09439337998628616, 0.054148342460393906, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19288603961467743, 0.11343871057033539, 0.04530506581068039, 0.08361206203699112, 0.05763270705938339, 0.11914192885160446, 0.11441229283809662, 0.12364516407251358, 0.07640955597162247, 0.07351644337177277, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.2517409026622772, 0.06019412353634834, 0.0493021197617054, 0.05922325327992439, 0.08453337103128433, 0.07307683676481247, 0.12308558821678162, 0.08293970674276352, 0.06380270421504974, 0.07185834646224976, 0.08024311810731888, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.24793605506420135, 0.048557717353105545, 0.06171693652868271, 0.06616786867380142, 0.08788600564002991, 0.059203874319791794, 0.09128590673208237, 0.06866814941167831, 0.060628652572631836, 0.08399353921413422, 0.0656941682100296, 0.05826117843389511, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15039606392383575, 0.07072296738624573, 0.0361880287528038, 0.03812887519598007, 0.054965775460004807, 0.09552796930074692, 0.1278020292520523, 0.09101754426956177, 0.041339464485645294, 0.06619426608085632, 0.11493086069822311, 0.0352681539952755, 0.07751806825399399, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21730749309062958, 0.0522487573325634, 0.042796872556209564, 0.05130990967154503, 0.07021071016788483, 0.060848668217659, 0.10094166547060013, 0.06729137897491455, 0.05347267910838127, 0.058032430708408356, 0.06386598944664001, 0.04143684729933739, 0.05429815128445625, 0.06593841314315796, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14158989489078522, 0.08735611289739609, 0.04109629988670349, 0.07771322876214981, 0.0664159432053566, 0.06084613874554634, 0.07890868932008743, 0.06467185169458389, 0.04156752675771713, 0.05314021185040474, 0.06033196300268173, 0.04990345984697342, 0.06147689372301102, 0.06152055040001869, 0.05346132814884186, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277396976947784, 0.05140271037817001, 0.034800730645656586, 0.044109929352998734, 0.04927105829119682, 0.08066022396087646, 0.09371130913496017, 0.07634278386831284, 0.03460073098540306, 0.05556498467922211, 0.09000372141599655, 0.027950916439294815, 0.058086879551410675, 0.09759876132011414, 0.0585305280983448, 0.03459075465798378, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.19136786460876465, 0.04588804021477699, 0.03718419373035431, 0.04478849098086357, 0.05845540761947632, 0.051623135805130005, 0.08451797068119049, 0.056617289781570435, 0.044779252260923386, 0.04681585356593132, 0.05177519842982292, 0.03474310785531998, 0.043655913323163986, 0.05257752537727356, 0.04571910202503204, 0.05523937940597534, 0.0542522631585598, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1780986338853836, 0.036513157188892365, 0.03537803143262863, 0.04660060629248619, 0.05606748163700104, 0.055425334721803665, 0.06580901890993118, 0.058695923537015915, 0.044602081179618835, 0.03845331072807312, 0.05522067844867706, 0.02812928892672062, 0.03887628763914108, 0.05635211616754532, 0.043072011321783066, 0.048592112958431244, 0.05801306664943695, 0.05610084533691406, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.11277122795581818, 0.04416341334581375, 0.01679084822535515, 0.03922899812459946, 0.030570218339562416, 0.08429384976625443, 0.05829833075404167, 0.079855777323246, 0.027740227058529854, 0.02460792288184166, 0.08568023890256882, 0.012135541066527367, 0.02122361958026886, 0.08979953825473785, 0.024395614862442017, 0.018000077456235886, 0.09797868877649307, 0.08564861118793488, 0.046817295253276825, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15648673474788666, 0.043593354523181915, 0.02946912869811058, 0.023382792249321938, 0.03989394009113312, 0.06601446866989136, 0.07239891588687897, 0.04959879070520401, 0.025720061734318733, 0.02872987650334835, 0.07268747687339783, 0.02010437287390232, 0.038949910551309586, 0.0779348760843277, 0.03185015916824341, 0.02957259863615036, 0.08162930607795715, 0.049830298870801926, 0.04757083207368851, 0.014582015573978424, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16675151884555817, 0.040216751396656036, 0.03194627910852432, 0.039155565202236176, 0.04867976903915405, 0.04351935163140297, 0.07027734071016312, 0.04676980897784233, 0.037370465695858, 0.03785952553153038, 0.04185350611805916, 0.029438795521855354, 0.03496702015399933, 0.04187152907252312, 0.036659594625234604, 0.04418948292732239, 0.042447034269571304, 0.04309000447392464, 0.05540826916694641, 0.024029972031712532, 0.04349849745631218, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.21311703324317932, 0.034521475434303284, 0.026512037962675095, 0.037284690886735916, 0.04200230911374092, 0.05378071963787079, 0.06410716474056244, 0.040195900946855545, 0.02951396070420742, 0.030562065541744232, 0.048140477389097214, 0.019645733758807182, 0.02385598234832287, 0.04712357744574547, 0.027805231511592865, 0.025632809847593307, 0.04690323397517204, 0.04211500659584999, 0.042453404515981674, 0.016209973022341728, 0.04720157012343407, 0.041315577924251556, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.16099101305007935, 0.031492140144109726, 0.02550598978996277, 0.03203630819916725, 0.0381193533539772, 0.05386464670300484, 0.05289202556014061, 0.06297370046377182, 0.03872942924499512, 0.030703019350767136, 0.048614975064992905, 0.016964145004749298, 0.020984875038266182, 0.04743489250540733, 0.026178553700447083, 0.03281702846288681, 0.04874100163578987, 0.03702305257320404, 0.05028446763753891, 0.014425939880311489, 0.04926920682191849, 0.027485143393278122, 0.052469126880168915, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13924194872379303, 0.04732196405529976, 0.022920073941349983, 0.03782676160335541, 0.037331584841012955, 0.05628196895122528, 0.06773108243942261, 0.06383747607469559, 0.02882455289363861, 0.028368638828396797, 0.05008406564593315, 0.014775182120501995, 0.021058909595012665, 0.04819711297750473, 0.017685111612081528, 0.015456225723028183, 0.04842803254723549, 0.03709954395890236, 0.042473260313272476, 0.018426815047860146, 0.0486132986843586, 0.02267395332455635, 0.0599895641207695, 0.025352953001856804, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1956234872341156, 0.032333292067050934, 0.02287408336997032, 0.038390789180994034, 0.03449879214167595, 0.04616904258728027, 0.05553748458623886, 0.03960667923092842, 0.023098403587937355, 0.02424929477274418, 0.040433045476675034, 0.016078541055321693, 0.021624963730573654, 0.03903055563569069, 0.026520874351263046, 0.028113430365920067, 0.03823631629347801, 0.03542612865567207, 0.041745759546756744, 0.021097134798765182, 0.03767954930663109, 0.03728693723678589, 0.044239211827516556, 0.02292151190340519, 0.03718467429280281, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0581965409219265, 0.038897402584552765, 0.026005256921052933, 0.023567914962768555, 0.022979985922574997, 0.05384150519967079, 0.058061059564352036, 0.06858772784471512, 0.03745982423424721, 0.029159115627408028, 0.05474672093987465, 0.013592944480478764, 0.02764706127345562, 0.05575402453541756, 0.03077986277639866, 0.016137346625328064, 0.05745428800582886, 0.03968729451298714, 0.03406554087996483, 0.013110976666212082, 0.05977657437324524, 0.016043763607740402, 0.02969929948449135, 0.020566541701555252, 0.08347289264202118, 0.03070863150060177, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.15561483800411224, 0.030323192477226257, 0.021067580208182335, 0.038600414991378784, 0.02508924901485443, 0.047195207327604294, 0.042556505650281906, 0.05166258290410042, 0.03036811575293541, 0.020133623853325844, 0.042556971311569214, 0.01851179078221321, 0.018671728670597076, 0.0414302833378315, 0.022022129967808723, 0.02106103114783764, 0.041666179895401, 0.027249203994870186, 0.04048159345984459, 0.015536422841250896, 0.042196206748485565, 0.017007360234856606, 0.062209486961364746, 0.015703151002526283, 0.05443187057971954, 0.04092410206794739, 0.015729248523712158, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1521887481212616, 0.032320477068424225, 0.02627762220799923, 0.032151371240615845, 0.0370805524289608, 0.03843334689736366, 0.042185209691524506, 0.03440922126173973, 0.02929617278277874, 0.024752285331487656, 0.034612320363521576, 0.021795086562633514, 0.023419689387083054, 0.03386644273996353, 0.028580596670508385, 0.034403245896101, 0.03315696492791176, 0.0267103910446167, 0.04213566333055496, 0.01902136206626892, 0.0327858068048954, 0.02683998830616474, 0.03304454684257507, 0.02330263890326023, 0.03390035405755043, 0.03944927453994751, 0.021334253251552582, 0.042546432465314865, 0.0, 0.0, 0.0, 0.0, 0.0], [0.10210394114255905, 0.048305343836545944, 0.027957916259765625, 0.028134865686297417, 0.04368893429636955, 0.03765961155295372, 0.040121808648109436, 0.03589581698179245, 0.024491779506206512, 0.01976805366575718, 0.03467409312725067, 0.01543145440518856, 0.032616566866636276, 0.03414338454604149, 0.027461379766464233, 0.02032320387661457, 0.03445841372013092, 0.01582757569849491, 0.030705859884619713, 0.02036094106733799, 0.03558045253157616, 0.0347902812063694, 0.03518148511648178, 0.02857258729636669, 0.04333386570215225, 0.03896390646696091, 0.017587684094905853, 0.04553378373384476, 0.04632509499788284, 0.0, 0.0, 0.0, 0.0], [0.1184173971414566, 0.02668977901339531, 0.028547650203108788, 0.035164568573236465, 0.03519093617796898, 0.030704861506819725, 0.042546890676021576, 0.023103510960936546, 0.025463121011853218, 0.029678545892238617, 0.028395216912031174, 0.019178254529833794, 0.023833557963371277, 0.02764001674950123, 0.03381751477718353, 0.039640139788389206, 0.02757442556321621, 0.02185617759823799, 0.03966766968369484, 0.03941833972930908, 0.02711561694741249, 0.030498065054416656, 0.03493281826376915, 0.032320536673069, 0.027840539813041687, 0.03877580910921097, 0.021124783903360367, 0.038611434400081635, 0.03277180343866348, 0.01947997882962227, 0.0, 0.0, 0.0], [0.05433792620897293, 0.019247835502028465, 0.011686460115015507, 0.014118616469204426, 0.013909779489040375, 0.05736804008483887, 0.0345463864505291, 0.04665059596300125, 0.013026435859501362, 0.013271292671561241, 0.05952776223421097, 0.009063143283128738, 0.01736420951783657, 0.06135563179850578, 0.012634643353521824, 0.015145537443459034, 0.06425533443689346, 0.04627062380313873, 0.01955636590719223, 0.012543467804789543, 0.06913556158542633, 0.034823667258024216, 0.03000595234334469, 0.01226181723177433, 0.07985590398311615, 0.01899644173681736, 0.010722930543124676, 0.086545430123806, 0.017174087464809418, 0.02214415930211544, 0.022453967481851578, 0.0, 0.0], [0.05849643424153328, 0.03437737748026848, 0.0136030912399292, 0.024170435965061188, 0.019005853682756424, 0.06546405702829361, 0.03062296472489834, 0.03573206812143326, 0.018859446048736572, 0.012241474352777004, 0.0628218874335289, 0.011835743673145771, 0.009552356787025928, 0.06431794911623001, 0.02158956602215767, 0.010912960395216942, 0.06665913015604019, 0.023997457697987556, 0.026596754789352417, 0.006153552792966366, 0.06984327733516693, 0.037238702178001404, 0.03004671260714531, 0.02168680727481842, 0.043640609830617905, 0.02120964229106903, 0.013490465469658375, 0.046327244490385056, 0.02043450064957142, 0.01336225401610136, 0.018568992614746094, 0.047140248119831085, 0.0], [0.1352868378162384, 0.02973336912691593, 0.026111334562301636, 0.03753575310111046, 0.0388815738260746, 0.0331667996942997, 0.04824388772249222, 0.030468309298157692, 0.02603820152580738, 0.024123704060912132, 0.02789273113012314, 0.022709932178258896, 0.024185284972190857, 0.02611454762518406, 0.02320965938270092, 0.025175275281071663, 0.02461479790508747, 0.021123144775629044, 0.02968505397439003, 0.018167467787861824, 0.023064393550157547, 0.02059808000922203, 0.03441140428185463, 0.021083392202854156, 0.02253883145749569, 0.032281555235385895, 0.013987245969474316, 0.024144919589161873, 0.027002302929759026, 0.012929439544677734, 0.03616538643836975, 0.02684023417532444, 0.03248501941561699]]]}\n",
                            "    )\n",
                            "    </script>"
                        ],
                        "text/plain": [
                            "<circuitsvis.utils.render.RenderedHTML at 0xffff115e44f0>"
                        ]
                    },
                    "execution_count": 304,
                    "metadata": {
                        "text/html": {
                            "Content-Type": "text/html"
                        }
                    },
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "print(\"Layer 0 Head Attention Patterns:\")\n",
                "cv.attention.attention_patterns(tokens=gpt2_str_tokens, attention=attention_pattern)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Hooks: Intervening on Activations"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "One of the great things about interpreting neural networks is that we have *full control* over our system. From a computational perspective, we know exactly what operations are going on inside (even if we don't know what they mean!). And we can make precise, surgical edits and see how the model's behaviour and other internals change. This is an extremely powerful tool, because it can let us eg set up careful counterfactuals and causal intervention to easily understand model behaviour. \n",
                "\n",
                "Accordingly, being able to do this is a pretty core operation, and this is one of the main things TransformerLens supports! The key feature here is **hook points**. Every activation inside the transformer is surrounded by a hook point, which allows us to edit or intervene on it. \n",
                "\n",
                "We do this by adding a **hook function** to that activation. The hook function maps `current_activation_value, hook_point` to `new_activation_value`. As the model is run, it computes that activation as normal, and then the hook function is applied to compute a replacement, and that is substituted in for the activation. The hook function can be an arbitrary Python function, so long as it returns a tensor of the correct shape.\n",
                "\n",
                "<details><summary>Relationship to PyTorch hooks</summary>\n",
                "\n",
                "[PyTorch hooks](https://blog.paperspace.com/pytorch-hooks-gradient-clipping-debugging/) are a great and underrated, yet incredibly janky, feature. They can act on a layer, and edit the input or output of that layer, or the gradient when applying autodiff. The key difference is that **Hook points** act on *activations* not layers. This means that you can intervene within a layer on each activation, and don't need to care about the precise layer structure of the transformer. And it's immediately clear exactly how the hook's effect is applied. This adjustment was shamelessly inspired by [Garcon's use of ProbePoints](https://transformer-circuits.pub/2021/garcon/index.html).\n",
                "\n",
                "They also come with a range of other quality of life improvements, like the model having a `model.reset_hooks()` method to remove all hooks, or helper methods to temporarily add hooks for a single forward pass - it is *incredibly* easy to shoot yourself in the foot with standard PyTorch hooks!\n",
                "</details>"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "As a basic example, let's [ablate](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=fh-HJyz1CgUVrXuoiban6bYx) head 7 in layer 0 on the text above. \n",
                "\n",
                "We define a `head_ablation_hook` function. This takes the value tensor for attention layer 0, and sets the component with `head_index==7` to zero and returns it (Note - we return by convention, but since we're editing the activation in-place, we don't strictly *need* to).\n",
                "\n",
                "We then use the `run_with_hooks` helper function to run the model and *temporarily* add in the hook for just this run. We enter in the hook as a tuple of the activation name (also the hook point name - found with `utils.get_act_name`) and the hook function."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 305,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Shape of the value tensor: torch.Size([1, 33, 12, 64])\n",
                        "Original Loss: 3.999\n",
                        "Ablated Loss: 5.453\n"
                    ]
                }
            ],
            "source": [
                "layer_to_ablate = 0\n",
                "head_index_to_ablate = 8\n",
                "\n",
                "# We define a head ablation hook\n",
                "# The type annotations are NOT necessary, they're just a useful guide to the reader\n",
                "# \n",
                "def head_ablation_hook(\n",
                "    value: Float[torch.Tensor, \"batch pos head_index d_head\"],\n",
                "    hook: HookPoint\n",
                ") -> Float[torch.Tensor, \"batch pos head_index d_head\"]:\n",
                "    print(f\"Shape of the value tensor: {value.shape}\")\n",
                "    value[:, :, head_index_to_ablate, :] = 0.\n",
                "    return value\n",
                "\n",
                "original_loss = model(gpt2_tokens, return_type=\"loss\")\n",
                "ablated_loss = model.run_with_hooks(\n",
                "    gpt2_tokens, \n",
                "    return_type=\"loss\", \n",
                "    fwd_hooks=[(\n",
                "        utils.get_act_name(\"v\", layer_to_ablate), \n",
                "        head_ablation_hook\n",
                "        )]\n",
                "    )\n",
                "print(f\"Original Loss: {original_loss.item():.3f}\")\n",
                "print(f\"Ablated Loss: {ablated_loss.item():.3f}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "**Gotcha:** Hooks are global state - they're added in as part of the model, and stay there until removed. `run_with_hooks` tries to create an abstraction where these are local state, by removing all hooks at the end of the function. But you can easily shoot yourself in the foot if there's, eg, an error in one of your hooks so the function never finishes. If you start getting bugs, try `model.reset_hooks()` to clean things up. Further, if you *do* add hooks of your own that you want to keep, which you can do with `add_perma_hook` on the relevant HookPoint"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Activation Patching on the Indirect Object Identification Task"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "For a somewhat more involved example, let's use hooks to apply **[activation patching](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=qeWBvs-R-taFfcCq-S_hgMqx)** on the **[Indirect Object Identification](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=iWsV3s5Kdd2ca3zNgXr5UPHa)** (IOI) task. \n",
                "\n",
                "The IOI task is the task of identifying that a sentence like \"After John and Mary went to the store, Mary gave a bottle of milk to\" continues with \" John\" rather than \" Mary\" (ie, finding the indirect object), and Redwood Research have [an excellent paper studying the underlying circuit in GPT-2 Small](https://arxiv.org/abs/2211.00593).\n",
                "\n",
                "**[Activation patching](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=qeWBvs-R-taFfcCq-S_hgMqx)** is a technique from [Kevin Meng and David Bau's excellent ROME paper](https://rome.baulab.info/). The goal is to identify which model activations are important for completing a task. We do this by setting up a **clean prompt** and a **corrupted prompt** and a **metric** for performance on the task. We then pick a specific model activation, run the model on the corrupted prompt, but then *intervene* on that activation and patch in its value when run on the clean prompt. We then apply the metric, and see how much this patch has recovered the clean performance. \n",
                "(See [a more detailed demonstration of activation patching here](https://colab.research.google.com/github.com/TransformerLensOrg/TransformerLens/blob/main/demos/Exploratory_Analysis_Demo.ipynb))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Here, our clean prompt is \"After John and Mary went to the store, **Mary** gave a bottle of milk to\", our corrupted prompt is \"After John and Mary went to the store, **John** gave a bottle of milk to\", and our metric is the difference between the correct logit ( John) and the incorrect logit ( Mary) on the final token. \n",
                "\n",
                "We see that the logit difference is significantly positive on the clean prompt, and significantly negative on the corrupted prompt, showing that the model is capable of doing the task!"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 306,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Clean logit difference: 4.276\n",
                        "Corrupted logit difference: -2.738\n"
                    ]
                }
            ],
            "source": [
                "clean_prompt = \"After John and Mary went to the store, Mary gave a bottle of milk to\"\n",
                "corrupted_prompt = \"After John and Mary went to the store, John gave a bottle of milk to\"\n",
                "\n",
                "clean_tokens = model.to_tokens(clean_prompt)\n",
                "corrupted_tokens = model.to_tokens(corrupted_prompt)\n",
                "\n",
                "def logits_to_logit_diff(logits, correct_answer=\" John\", incorrect_answer=\" Mary\"):\n",
                "    # model.to_single_token maps a string value of a single token to the token index for that token\n",
                "    # If the string is not a single token, it raises an error.\n",
                "    correct_index = model.to_single_token(correct_answer)\n",
                "    incorrect_index = model.to_single_token(incorrect_answer)\n",
                "    return logits[0, -1, correct_index] - logits[0, -1, incorrect_index]\n",
                "\n",
                "# We run on the clean prompt with the cache so we store activations to patch in later.\n",
                "clean_logits, clean_cache = model.run_with_cache(clean_tokens)\n",
                "clean_logit_diff = logits_to_logit_diff(clean_logits)\n",
                "print(f\"Clean logit difference: {clean_logit_diff.item():.3f}\")\n",
                "\n",
                "# We don't need to cache on the corrupted prompt.\n",
                "corrupted_logits = model(corrupted_tokens)\n",
                "corrupted_logit_diff = logits_to_logit_diff(corrupted_logits)\n",
                "print(f\"Corrupted logit difference: {corrupted_logit_diff.item():.3f}\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We now setup the hook function to do activation patching. Here, we'll patch in the [residual stream](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=DHp9vZ0h9lA9OCrzG2Y3rrzH) at the start of a specific layer and at a specific position. This will let us see how much the model is using the residual stream at that layer and position to represent the key information for the task. \n",
                "\n",
                "We want to iterate over all layers and positions, so we write the hook to take in an position parameter. Hook functions must have the input signature (activation, hook), but we can use `functools.partial` to set the position parameter before passing it to `run_with_hooks`"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 307,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "980e183587f54a03bb4ead134831c94d",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/12 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# We define a residual stream patching hook\n",
                "# We choose to act on the residual stream at the start of the layer, so we call it resid_pre\n",
                "# The type annotations are a guide to the reader and are not necessary\n",
                "def residual_stream_patching_hook(\n",
                "    resid_pre: Float[torch.Tensor, \"batch pos d_model\"],\n",
                "    hook: HookPoint,\n",
                "    position: int\n",
                ") -> Float[torch.Tensor, \"batch pos d_model\"]:\n",
                "    # Each HookPoint has a name attribute giving the name of the hook.\n",
                "    clean_resid_pre = clean_cache[hook.name]\n",
                "    resid_pre[:, position, :] = clean_resid_pre[:, position, :]\n",
                "    return resid_pre\n",
                "\n",
                "# We make a tensor to store the results for each patching run. We put it on the model's device to avoid needing to move things between the GPU and CPU, which can be slow.\n",
                "num_positions = len(clean_tokens[0])\n",
                "ioi_patching_result = torch.zeros((model.cfg.n_layers, num_positions), device=model.cfg.device)\n",
                "\n",
                "for layer in tqdm.tqdm(range(model.cfg.n_layers)):\n",
                "    for position in range(num_positions):\n",
                "        # Use functools.partial to create a temporary hook function with the position fixed\n",
                "        temp_hook_fn = partial(residual_stream_patching_hook, position=position)\n",
                "        # Run the model with the patching hook\n",
                "        patched_logits = model.run_with_hooks(corrupted_tokens, fwd_hooks=[\n",
                "            (utils.get_act_name(\"resid_pre\", layer), temp_hook_fn)\n",
                "        ])\n",
                "        # Calculate the logit difference\n",
                "        patched_logit_diff = logits_to_logit_diff(patched_logits).detach()\n",
                "        # Store the result, normalizing by the clean and corrupted logit difference so it's between 0 and 1 (ish)\n",
                "        ioi_patching_result[layer, position] = (patched_logit_diff - corrupted_logit_diff)/(clean_logit_diff - corrupted_logit_diff)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can now visualize the results, and see that this computation is extremely localised within the model. Initially, the second subject (Mary) token is all that matters (naturally, as it's the only different token), and all relevant information remains here until heads in layer 7 and 8 move this to the final token where it's used to predict the indirect object.\n",
                "(Note - the heads are in layer 7 and 8, not 8 and 9, because we patched in the residual stream at the *start* of each layer)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 308,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"1d04e6ee-31ed-4e10-b7a1-08f454ff13b2\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1d04e6ee-31ed-4e10-b7a1-08f454ff13b2\")) {                    Plotly.newPlot(                        \"1d04e6ee-31ed-4e10-b7a1-08f454ff13b2\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"x\":[\"\\u003c|endoftext|\\u003e_0\",\"After_1\",\" John_2\",\" and_3\",\" Mary_4\",\" went_5\",\" to_6\",\" the_7\",\" store_8\",\",_9\",\" Mary_10\",\" gave_11\",\" a_12\",\" bottle_13\",\" of_14\",\" milk_15\",\" to_16\"],\"z\":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9981480240821838,0.0016014170832931995,0.00014983542496338487,-0.00037064551725052297,-2.1482755983015522e-05,-0.000627894711215049,-0.0005147703341208398],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9980558156967163,0.0022839705925434828,0.0001830113324103877,-0.000504164956510067,-0.0002675826835911721,-5.1395454647718e-05,-0.0012810792541131377],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9967373609542847,0.004082539584487677,0.000974066206254065,4.3509378883754835e-05,-0.0001593531051184982,-0.0003361099516041577,-0.0019437815062701702],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9905893206596375,0.019987665116786957,0.001896193134598434,0.0010143123799934983,-6.716760253766552e-05,0.0009109776583500206,-0.0019008159870281816],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9616512656211853,0.08534800261259079,0.005204265471547842,0.0030527268536388874,0.00019687994790729135,0.001105954055674374,-0.002283698646351695],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9630993008613586,0.08437121659517288,0.004121969919651747,0.0007184486021287739,0.000102790909295436,0.0010028912220150232,-0.004215243272483349],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.9359176158905029,0.11111806333065033,0.007705239113420248,0.000375812262063846,0.00036575071862898767,0.001326492172665894,0.018744928762316704],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.7701548933982849,0.03741942718625069,0.0020680550951510668,-8.239589078584686e-05,0.00013460713671520352,0.001724603003822267,0.44990602135658264],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.09650597721338272,0.025926152244210243,0.001972062513232231,0.0003298554802313447,0.00042503225267864764,0.0018855876987800002,0.8994725346565247],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.023322386667132378,0.018538258969783783,0.0015875485260039568,0.0005272792768664658,0.0002534421219024807,0.0008737227180972695,0.9612759947776794],[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,-0.008558566682040691,0.006340676452964544,0.0005816660122945905,-0.0003418205596972257,0.00011094891669927165,0.0006477459101006389,0.9495818614959717]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Position: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Position\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('1d04e6ee-31ed-4e10-b7a1-08f454ff13b2');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# Add the index to the end of the label, because plotly doesn't like duplicate labels\n",
                "token_labels = [f\"{token}_{index}\" for index, token in enumerate(model.to_str_tokens(clean_tokens))]\n",
                "imshow(ioi_patching_result, x=token_labels, xaxis=\"Position\", yaxis=\"Layer\", title=\"Normalized Logit Difference After Patching Residual Stream on the IOI Task\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Hooks: Accessing Activations"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Hooks can also be used to just **access** an activation - to run some function using that activation value, *without* changing the activation value. This can be achieved by just having the hook return nothing, and not editing the activation in place. \n",
                "\n",
                "This is useful for eg extracting activations for a specific task, or for doing some long-running calculation across many inputs, eg finding the text that most activates a specific neuron. (Note - everything this can do *could* be done with `run_with_cache` and post-processing, but this workflow can be more intuitive and memory efficient.)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "To demonstrate this, let's look for **[induction heads](https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html)** in GPT-2 Small. \n",
                "\n",
                "Induction circuits are a very important circuit in generative language models, which are used to detect and continue repeated subsequences. They consist of two heads in separate layers that compose together, a **previous token head** which always attends to the previous token, and an **induction head** which attends to the token *after* an earlier copy of the current token. \n",
                "\n",
                "To see why this is important, let's say that the model is trying to predict the next token in a news article about Michael Jordan. The token \" Michael\", in general, could be followed by many surnames. But an induction head will look from that occurence of \" Michael\" to the token after previous occurences of \" Michael\", ie \" Jordan\" and can confidently predict that that will come next."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "An interesting fact about induction heads is that they generalise to arbitrary sequences of repeated tokens. We can see this by generating sequences of 50 random tokens, repeated twice, and plotting the average loss at predicting the next token, by position. We see that the model goes from terrible to very good at the halfway point."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 309,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"d3e84661-e12c-4a84-8655-2f33ba1284da\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d3e84661-e12c-4a84-8655-2f33ba1284da\")) {                    Plotly.newPlot(                        \"d3e84661-e12c-4a84-8655-2f33ba1284da\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98],\"xaxis\":\"x\",\"y\":[10.872052192687988,12.263540267944336,14.253204345703125,13.246556282043457,13.005993843078613,13.460993766784668,12.97065258026123,13.064518928527832,11.797075271606445,12.513029098510742,12.812576293945312,12.767385482788086,12.285192489624023,13.085138320922852,12.94897747039795,12.297746658325195,12.90766716003418,11.799567222595215,12.414271354675293,12.327254295349121,10.539899826049805,12.742929458618164,11.173940658569336,11.951539993286133,12.244311332702637,13.201930046081543,11.1293363571167,11.558095932006836,11.154999732971191,10.811470031738281,11.701059341430664,11.395160675048828,12.566507339477539,11.18510627746582,10.747629165649414,11.653327941894531,12.03685188293457,11.461939811706543,12.074749946594238,9.762995719909668,11.312043190002441,12.081087112426758,11.683740615844727,11.636404991149902,13.009252548217773,11.74526596069336,11.57121753692627,11.874940872192383,11.46745777130127,10.957990646362305,1.9599711894989014,0.8824893236160278,0.8781511187553406,0.3232020437717438,0.4251673221588135,0.25897061824798584,0.4066997468471527,0.21202726662158966,0.16516722738742828,0.11655920743942261,0.2388218641281128,0.04333684220910072,0.14030048251152039,0.1186639666557312,0.18781667947769165,0.05167889595031738,0.07741443812847137,0.08716835081577301,0.07370562106370926,0.04441319778561592,0.06953944265842438,0.030275847762823105,0.03782800957560539,0.04936334118247032,0.07152687013149261,0.11301679909229279,0.03093745745718479,0.05099482089281082,0.034429892897605896,0.010404873639345169,0.10524864494800568,0.14771051704883575,0.013218658044934273,0.1266184151172638,0.0512910857796669,0.007522976957261562,0.11803790181875229,0.1785733997821808,0.13207308948040009,0.32432517409324646,0.19658631086349487,0.07713610678911209,0.03152012079954147,0.05610422044992447,0.029515912756323814,0.02569529041647911,0.0804244875907898,0.0179621372371912,0.07805689424276352],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Loss by position on random repeated tokens\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('d3e84661-e12c-4a84-8655-2f33ba1284da');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "batch_size = 10\n",
                "seq_len = 50\n",
                "size = (batch_size, seq_len)\n",
                "input_tensor = torch.randint(1000, 10000, size)\n",
                "\n",
                "random_tokens = input_tensor.to(model.cfg.device)\n",
                "repeated_tokens = einops.repeat(random_tokens, \"batch seq_len -> batch (2 seq_len)\")\n",
                "repeated_logits = model(repeated_tokens)\n",
                "correct_log_probs = model.loss_fn(repeated_logits, repeated_tokens, per_token=True)\n",
                "loss_by_position = einops.reduce(correct_log_probs, \"batch position -> position\", \"mean\")\n",
                "line(loss_by_position, xaxis=\"Position\", yaxis=\"Loss\", title=\"Loss by position on random repeated tokens\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The induction heads will be attending from the second occurence of each token to the token *after* its first occurence, ie the token `50-1==49` places back. So by looking at the average attention paid 49 tokens back, we can identify induction heads! Let's define a hook to do this!\n",
                "\n",
                "<details><summary>Technical details</summary>\n",
                "\n",
                "* We attach the hook to the attention pattern activation. There's one big pattern activation per layer, stacked across all heads, so we need to do some tensor manipulation to get a per-head score. \n",
                "* Hook functions can access global state, so we make a big tensor to store the induction head score for each head, and then we just add the score for each head to the appropriate position in the tensor. \n",
                "* To get a single hook function that works for each layer, we use the `hook.layer()` method to get the layer index (internally this is just inferred from the hook names).\n",
                "* As we want to add this to *every* activation pattern hook point, rather than giving the string for an activation name, this time we give a **name filter**. This is a Boolean function on hook point names, and it adds the hook function to every hook point where the function evaluates as true. \n",
                "    * `run_with_hooks` allows us to enter a list of (act_name, hook_function) pairs to all be added at once, so we could also have done this by inputting a list with a hook for each layer.\n",
                "</details>"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 310,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"190ab42e-4456-4833-a298-6f585a1583e4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"190ab42e-4456-4833-a298-6f585a1583e4\")) {                    Plotly.newPlot(                        \"190ab42e-4456-4833-a298-6f585a1583e4\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.009955878369510174,9.966958168661222e-05,0.010546973906457424,4.0583410054750857e-07,0.00022813043324276805,0.00019768899073824286,0.00977946724742651,0.0006674872129224241,0.00908320676535368,0.00915559846907854,0.006828702986240387,0.015242615714669228],[0.0011627675266936421,0.00045248764217831194,0.0021361575927585363,0.01417490839958191,0.004926139954477549,0.010640118271112442,0.015927109867334366,0.013070465996861458,0.012896527536213398,0.016126573085784912,0.006526595447212458,0.000517632404807955],[0.0044746194034814835,0.01877586357295513,0.003041735850274563,0.0019074507290497422,0.012237360700964928,0.002584304893389344,0.0039876471273601055,0.008248553611338139,0.004771647043526173,0.0017633740790188313,0.0006906316848471761,0.010382029227912426],[0.015542104840278625,0.007110548205673695,0.002217961009591818,0.012544052675366402,0.021509619429707527,0.011987114325165749,0.00174334819894284,0.0009274697513319552,0.005935505498200655,0.01233423687517643,0.009232483804225922,0.006464063189923763],[0.01664578728377819,0.014668889343738556,0.013992691412568092,0.008478098548948765,0.01899615488946438,0.012697082944214344,0.008333329111337662,0.0017152393702417612,0.01665916107594967,0.014090241864323616,0.018890956416726112,8.906972381872436e-11],[0.451614111661911,0.9170698523521423,0.01424565352499485,0.0065706996247172356,0.011130396276712418,0.9321417212486267,0.008963020518422127,0.018100803717970848,0.02871188521385193,0.029120853170752525,0.02137076109647751,0.01733485981822014],[0.008789685554802418,0.017398007214069366,0.01834423653781414,0.015153961256146431,0.02248268947005272,0.011032403446733952,0.03012258931994438,0.01068038959056139,0.009857730939984322,0.9169892072677612,0.036462243646383286,0.01406988874077797],[0.01107383705675602,0.17768746614456177,0.8614926934242249,0.019131189212203026,0.018100876361131668,0.016298996284604073,0.04784739762544632,0.08873092383146286,0.01723749376833439,0.019047973677515984,0.9243332743644714,0.06009266525506973],[0.015750497579574585,0.40704065561294556,0.014495478942990303,0.050178349018096924,0.017805716022849083,0.012079598382115364,0.15166832506656647,0.013180013746023178,0.032852329313755035,0.03178909420967102,0.06760691851377487,0.022921957075595856],[0.25689446926116943,0.19054049253463745,0.10555234551429749,0.012684683315455914,0.0927107185125351,0.026271803304553032,0.4618716239929199,0.029983991757035255,0.05182349309325218,0.4789004623889923,0.016641730442643166,0.03995012864470482],[0.339070200920105,0.5105082392692566,0.038450296968221664,0.14799615740776062,0.05797654390335083,0.01543364953249693,0.3008923828601837,0.47816023230552673,0.05431150645017624,0.015494456514716148,0.16141793131828308,0.2569926977157593],[0.017057929188013077,0.053864460438489914,0.03378748893737793,0.009234139695763588,0.03453892469406128,0.1011928990483284,0.04960957169532776,0.07048070430755615,0.009171898476779461,0.30352190136909485,0.40838193893432617,0.022866230458021164]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('190ab42e-4456-4833-a298-6f585a1583e4');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# We make a tensor to store the induction score for each head. We put it on the model's device to avoid needing to move things between the GPU and CPU, which can be slow.\n",
                "induction_score_store = torch.zeros((model.cfg.n_layers, model.cfg.n_heads), device=model.cfg.device)\n",
                "def induction_score_hook(\n",
                "    pattern: Float[torch.Tensor, \"batch head_index dest_pos source_pos\"],\n",
                "    hook: HookPoint,\n",
                "):\n",
                "    # We take the diagonal of attention paid from each destination position to source positions seq_len-1 tokens back\n",
                "    # (This only has entries for tokens with index>=seq_len)\n",
                "    induction_stripe = pattern.diagonal(dim1=-2, dim2=-1, offset=1-seq_len)\n",
                "    # Get an average score per head\n",
                "    induction_score = einops.reduce(induction_stripe, \"batch head_index position -> head_index\", \"mean\")\n",
                "    # Store the result.\n",
                "    induction_score_store[hook.layer(), :] = induction_score\n",
                "\n",
                "# We make a boolean filter on activation names, that's true only on attention pattern names.\n",
                "pattern_hook_names_filter = lambda name: name.endswith(\"pattern\")\n",
                "\n",
                "model.run_with_hooks(\n",
                "    repeated_tokens, \n",
                "    return_type=None, # For efficiency, we don't need to calculate the logits\n",
                "    fwd_hooks=[(\n",
                "        pattern_hook_names_filter,\n",
                "        induction_score_hook\n",
                "    )]\n",
                ")\n",
                "\n",
                "imshow(induction_score_store, xaxis=\"Head\", yaxis=\"Layer\", title=\"Induction Score by Head\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Head 5 in Layer 5 scores extremely highly on this score, and we can feed in a shorter repeated random sequence, visualize the attention pattern for it and see this directly - including the \"induction stripe\" at `seq_len-1` tokens back.\n",
                "\n",
                "This time we put in a hook on the attention pattern activation to visualize the pattern of the relevant head."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 311,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div id=\"circuits-vis-fcf89ad6-45d6\" style=\"margin: 15px 0;\"/>\n",
                            "    <script crossorigin type=\"module\">\n",
                            "    import { render, AttentionPatterns } from \"https://unpkg.com/circuitsvis@1.43.0/dist/cdn/esm.js\";\n",
                            "    render(\n",
                            "      \"circuits-vis-fcf89ad6-45d6\",\n",
                            "      AttentionPatterns,\n",
                            "      {\"tokens\": [\"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\", \"use\", \" advice\", \" Social\", \"\\u00f6\", \"\\u00b7\", \" fought\", \" Le\", \" allegedly\", \" NO\", \"alth\", \"car\", \" prepared\", \"new\", \"rant\", \"roll\", \" hours\", \" published\", \"66\", \"ension\", \" 44\"], \"attention\": [[[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9737270474433899, 0.0262729711830616, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9820428490638733, 0.017020266503095627, 0.0009368456667289138, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9895542860031128, 0.00866580568253994, 0.0004119748482480645, 0.0013679902767762542, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8543053865432739, 0.0780181884765625, 0.0008415378979407251, 0.00013599172234535217, 0.06669897586107254, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9374335408210754, 0.033002182841300964, 0.0015577428275719285, 2.5352785542054335e-06, 0.0010925536043941975, 0.026911530643701553, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.976921021938324, 0.0038436956238001585, 2.234029489045497e-05, 3.521895996527746e-05, 0.005183499306440353, 0.01217629387974739, 0.0018179028993472457, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9473506212234497, 0.013174930587410927, 0.0013492131838575006, 1.180242543341592e-05, 0.0009449435747228563, 0.011318957433104515, 0.018021011725068092, 0.007828536443412304, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847127199172974, 0.0010781448800116777, 0.002173440996557474, 5.48224352314719e-06, 0.0004914223100058734, 0.0013570792507380247, 0.0001018581388052553, 0.00028538500191643834, 0.009794448502361774, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9915198683738708, 0.0044833519496023655, 0.00012727153080049902, 0.0001670209167059511, 0.0016301727155223489, 0.0011521612759679556, 0.0003231288574170321, 0.00012646260438486934, 0.00039313812158070505, 7.735053804935887e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8908807635307312, 0.024311939254403114, 1.7341229977319017e-05, 4.1577197407605127e-05, 0.0008967601461336017, 0.07334909588098526, 0.0009482800960540771, 0.004280842375010252, 0.005168660078197718, 7.830293725419324e-06, 9.693214815342799e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.817081093788147, 0.13517087697982788, 0.011989914812147617, 1.1421690032875631e-05, 0.0003511958639137447, 0.00945067685097456, 0.01946328766644001, 0.0006557486485689878, 0.0005761014763265848, 2.9927012292318977e-05, 1.658979817875661e-05, 0.005203105043619871, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9082697033882141, 0.006068143527954817, 0.013871830888092518, 0.0008237074362114072, 0.011908311396837234, 0.01554207131266594, 0.008354817517101765, 0.0020781648345291615, 0.0013173273764550686, 0.0021398114040493965, 0.003944162279367447, 0.0012376609956845641, 0.02444424293935299, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9479592442512512, 0.0021026760805398226, 0.01193847507238388, 0.00012338522356003523, 3.537495786076761e-06, 0.00014498857490252703, 0.0005875465576536953, 2.5534713131492026e-05, 0.0013609088491648436, 0.0003395720850676298, 0.01007620245218277, 0.0157905463129282, 0.006346344482153654, 0.003201034851372242, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9393549561500549, 0.006392289884388447, 0.0018427352188155055, 6.116198164818343e-06, 0.0003358719404786825, 0.0020515176001936197, 0.003801520448178053, 0.0012357976520434022, 0.0002194812404923141, 0.0003869338543154299, 5.012214751332067e-05, 0.008153197355568409, 0.026924636214971542, 0.002938011661171913, 0.0063067772425711155, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9339620471000671, 0.0017828113632276654, 0.005864645820111036, 0.000199502072064206, 7.227147580124438e-05, 0.001453535514883697, 0.0025924306828528643, 0.0004859396431129426, 0.002229833509773016, 0.00015120525495149195, 0.012292936444282532, 0.005057854112237692, 0.012368598021566868, 0.003944497089833021, 0.0062751127406954765, 0.011266660876572132, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8931334614753723, 0.0015468199271708727, 0.013001665472984314, 7.96635686128866e-06, 5.864337435923517e-05, 0.0008863371913321316, 0.0032020823564380407, 3.214758908143267e-05, 0.00018022512085735798, 1.1455734238552395e-05, 7.600105163874105e-05, 0.0004202726122457534, 0.001612616004422307, 0.028539085760712624, 0.010535502806305885, 0.025432026013731956, 0.021323613822460175, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.9847024083137512, 0.00045824647531844676, 0.0001722048327792436, 6.160975090097054e-07, 4.7827966227487195e-06, 0.0005806126864627004, 0.00044618724496103823, 0.00041201553540304303, 0.0013038743054494262, 0.00031760730780661106, 6.99415395502001e-05, 0.0013941085198894143, 5.5830587371019647e-05, 0.0009110421524383128, 0.0001955802144948393, 0.000396028597606346, 0.0011691706022247672, 0.007409737445414066, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8922598958015442, 0.010283716022968292, 0.007569305133074522, 0.015225780196487904, 0.000603529391810298, 0.0014377714833244681, 0.018397411331534386, 0.000181866911589168, 0.0021135706920176744, 3.8036650948924944e-05, 0.009962501004338264, 0.003998196218162775, 0.0012666822876781225, 0.002186268102377653, 0.003267065854743123, 0.0015871906653046608, 0.019133716821670532, 0.008779392577707767, 0.0017080693505704403, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.5637850761413574, 2.2041742340661585e-05, 0.00038083098479546607, 1.3938017673353897e-07, 4.306914647145277e-08, 0.0001288325438508764, 5.202714601182379e-05, 4.098215413250728e-06, 0.00043821678264066577, 1.0102498890773859e-05, 2.0490140741458163e-05, 0.00021747533173765987, 2.5249997634091415e-05, 2.1293963072821498e-05, 0.002207203535363078, 5.8927667851094157e-05, 0.002418374177068472, 0.0032775455620139837, 0.4260479211807251, 0.0008841017843224108, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.14634987711906433, 0.4510916769504547, 0.027205228805541992, 0.003008269937708974, 0.0007913715671747923, 0.08009303361177444, 0.005927639082074165, 0.0006846353644505143, 0.0021268511191010475, 0.0027747598942369223, 0.00023907265858724713, 0.002550537697970867, 0.005493414122611284, 0.015832215547561646, 0.0003449993673712015, 0.0005726668750867248, 0.0021751606836915016, 0.03904319554567337, 0.1698266863822937, 0.041207652539014816, 0.0026610144414007664, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.1588301807641983, 0.09413877129554749, 0.6926900744438171, 4.7764006012585014e-05, 8.085336048679892e-06, 0.009355566464364529, 0.0008445510757155716, 2.443790663164691e-06, 0.0001377410371787846, 1.1189789574928e-06, 4.677354354498675e-06, 0.0003472109674476087, 0.0026314801070839167, 0.0004504164680838585, 0.006463209632784128, 0.0005723321228288114, 0.001266839331947267, 0.006402328610420227, 0.0018093092367053032, 0.006555440369993448, 0.0003791518392972648, 0.01706133596599102, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0507238395512104, 0.004356134217232466, 0.00013167195720598102, 0.93964684009552, 0.0005500880652107298, 0.002771187573671341, 1.4556246242136694e-05, 5.017395324102836e-06, 1.5498708307859488e-05, 7.02077898040443e-08, 8.694883035786916e-06, 3.654152897070162e-05, 3.6079711662750924e-06, 2.594179386505857e-05, 7.59087424739846e-06, 7.100912853275076e-07, 4.6297875087475404e-05, 7.143527909647673e-05, 0.00012089155643479899, 0.000561000662855804, 1.3380984455579892e-05, 0.0007342093158513308, 0.00015471279039047658, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04466324299573898, 0.0028326697647571564, 7.648682367289439e-05, 0.00015513764810748398, 0.7502217888832092, 0.1919575184583664, 9.640491043683141e-05, 0.00016210104513447732, 0.00012769455497618765, 1.1226586138946004e-05, 8.733231879887171e-06, 0.0002813311293721199, 5.207761569181457e-05, 0.008386502042412758, 4.340233772381907e-06, 6.482672324636951e-05, 3.802950232056901e-05, 7.603670383105054e-05, 0.00012636416067834944, 9.22799008549191e-05, 4.0301836179423844e-07, 0.00011281618208158761, 2.522413069527829e-06, 0.00044938692008145154, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0136062391102314, 0.006971819791942835, 3.188595292158425e-05, 1.8455398276273627e-06, 0.0023010680451989174, 0.9711790680885315, 0.0003632439475040883, 8.45976173877716e-05, 0.00010611514881020412, 4.505663468989951e-07, 2.987568166190613e-07, 0.0008592635276727378, 8.84485270944424e-05, 0.0003481197636574507, 9.285117812396493e-07, 3.160546111757867e-05, 1.2802072888007388e-05, 5.803379099234007e-05, 0.00010517534974496812, 6.438309355871752e-05, 1.8867468725147774e-06, 0.0009238768252544105, 4.681064638134558e-06, 8.993229130282998e-06, 0.002845223993062973, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.019611306488513947, 0.004386154469102621, 6.198722985573113e-05, 4.885768589701911e-08, 4.2524367017904297e-05, 0.0036122521851211786, 0.9598399996757507, 0.005622244905680418, 0.002344567561522126, 5.173993713469827e-07, 1.962153874046635e-06, 0.0016548263374716043, 0.0005915339570492506, 0.001169586437754333, 5.784231689176522e-06, 8.118995174299926e-05, 4.500300929066725e-05, 0.0001849783438956365, 5.086977398605086e-05, 0.000111328401544597, 6.848466000519693e-06, 3.85396160709206e-05, 4.853071914112661e-06, 1.2763491952227923e-07, 3.953082341467962e-06, 0.0005269552930258214, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.008729123510420322, 0.00017810783174354583, 2.8806312002416234e-07, 4.138274221077154e-07, 6.864992610644549e-05, 0.0009331199107691646, 0.0001488830748712644, 0.9844523668289185, 0.004786375444382429, 0.0001132896650233306, 7.25545532986871e-07, 7.424702926073223e-05, 1.4996358004282229e-05, 0.00019790712394751608, 2.995068371092202e-07, 4.872013960266486e-06, 5.296111794450553e-06, 4.536831511359196e-06, 0.00011268968228250742, 3.4171025617979467e-06, 8.272540071629919e-06, 1.237986271007685e-05, 3.665547154696469e-08, 9.025117719829723e-07, 1.6053079889388755e-05, 0.00011404424003558233, 1.8653661754797213e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.04082169756293297, 0.003618433838710189, 4.8210502427536994e-05, 1.9615818303009291e-07, 5.380281072575599e-05, 0.0021802405826747417, 0.003327243495732546, 0.0018782124388962984, 0.9307870268821716, 0.004815405700355768, 1.4430276678467635e-05, 0.0028288080357015133, 0.0001046408069669269, 0.0029309175442904234, 0.0009514765115454793, 6.522196053992957e-05, 0.0002954130177386105, 0.00012317558866925538, 0.001551239751279354, 0.0005327718099579215, 0.00022083611111156642, 0.0004432721179910004, 1.939156027219724e-05, 5.825709763485065e-07, 1.663282819208689e-05, 0.0004773263353854418, 0.0011258200975134969, 0.000767689838539809, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.13800935447216034, 0.0003385838062968105, 0.00015972652181517333, 3.422568184419106e-08, 1.2696329577011056e-05, 0.00024196661252062768, 3.8219157431740314e-05, 3.751519398065284e-05, 0.004743278957903385, 0.8406777381896973, 0.0001887540565803647, 0.00015232608711812645, 1.2507432074926328e-05, 0.0002236285072285682, 0.00013282443978823721, 6.470834341598675e-05, 0.00013950421998742968, 9.763532580109313e-05, 0.0004387960070744157, 4.4860902562504634e-05, 6.358889368129894e-05, 0.00034255790524184704, 5.2226645493647084e-05, 5.293432536745968e-07, 4.435891696630279e-06, 6.54347168165259e-05, 2.914482593041612e-06, 3.872663910442498e-06, 0.013709748163819313, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.007798762992024422, 0.00022834629635326564, 4.6384946017496986e-07, 6.367677087837365e-07, 1.5816745872143656e-05, 4.7439083573408425e-05, 5.239070105744759e-06, 7.306558018171927e-06, 1.2522126780822873e-05, 3.3302333690699015e-07, 0.9909055233001709, 0.00045220478205010295, 5.365327069739578e-06, 7.535887561971322e-05, 8.799969691608567e-06, 7.89504611020675e-06, 0.00023918120132293552, 2.256896095786942e-06, 7.641861884621903e-05, 5.5141779739642516e-05, 4.378313406050438e-06, 2.181060699513182e-05, 4.4351477157533736e-08, 1.258427687389485e-06, 2.069825995931751e-06, 1.2951696589880157e-05, 3.9528120510112785e-07, 1.1202276937183342e-06, 7.9817673395155e-06, 2.9978505153849255e-06, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.042284343391656876, 0.011305336840450764, 7.640699664079875e-07, 2.2586050363315735e-06, 9.786370355868712e-05, 0.038269199430942535, 0.00023640785366296768, 0.0014555181842297316, 0.0029497782234102488, 1.2863498568549403e-06, 2.827169737429358e-05, 0.8946184515953064, 0.000523496069945395, 0.001179973711259663, 0.0009106355137191713, 0.00036046106833964586, 0.00020461619715206325, 2.910214607254602e-05, 0.0013425356009975076, 0.0003398243279661983, 0.00033255034941248596, 8.43034649733454e-05, 7.937682511283128e-08, 5.809831236547325e-06, 4.910861207463313e-06, 0.0023872887250036, 2.6967012672685087e-05, 0.00020931517065037042, 0.0007688838522881269, 5.801371116831433e-06, 3.4147673432016745e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.006102928426116705, 0.016550440341234207, 4.6485070924973115e-05, 8.233602954987873e-08, 7.398419256787747e-06, 0.0005482262931764126, 0.000593138684052974, 1.573847112013027e-05, 3.5032899177167565e-05, 3.3504232987979776e-07, 1.8657554790024733e-07, 0.000460667914012447, 0.8416212201118469, 0.12311417609453201, 0.00635922746732831, 0.002699504140764475, 0.00016676213999744505, 0.0012130774557590485, 0.00013198891247157007, 8.468204759992659e-05, 4.615942543750862e-06, 6.99491283739917e-05, 4.916110356134595e-06, 3.4761166034513735e-07, 3.71195625348264e-07, 6.107363878982142e-05, 6.7285327531863e-05, 2.275831548104179e-06, 2.060149927274324e-05, 7.198155458354449e-07, 2.7881066344548344e-08, 1.652937862672843e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0404171422123909, 0.0013464416842907667, 0.0002881725085899234, 4.343583896115888e-06, 0.0006837462424300611, 0.0021492778323590755, 0.0012713986216112971, 0.00021145936625543982, 0.00014913539052940905, 2.368190689594485e-05, 9.255170880351216e-05, 7.46091318433173e-05, 0.0030381649266928434, 0.9153454899787903, 0.0020634918473660946, 0.002670771675184369, 0.0006971447728574276, 0.022915314882993698, 0.0016386568313464522, 0.00029437849298119545, 8.30199132906273e-06, 0.00011940939293708652, 3.610887142713182e-05, 1.2593977771757636e-05, 0.000383078760933131, 0.0011952179484069347, 0.00018445361638441682, 7.125888805603608e-05, 0.00012850709026679397, 6.062284592189826e-05, 4.517687557381578e-05, 6.895808724038943e-07, 0.0023790940176695585, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.03775651007890701, 0.0016132266027852893, 0.0003177846665494144, 8.171087984010228e-07, 4.6314994506246876e-07, 7.222019485197961e-05, 0.00012490902736317366, 4.997585165256169e-06, 0.0003066718054469675, 5.749355295847636e-06, 0.0002466128207743168, 0.003777747042477131, 0.0013358069118112326, 0.002113186754286289, 0.8997160792350769, 0.04417850449681282, 0.00033094992977567017, 0.001384939532727003, 0.0003935607383027673, 0.002321055391803384, 0.0004642207932192832, 0.00019271507335361093, 0.0005703868810087442, 5.92765218243585e-06, 2.4687210498086642e-08, 3.108325699940906e-06, 7.16592330718413e-06, 2.9240155186016636e-07, 7.98414257587865e-05, 1.2546398465929087e-05, 0.0001265132159460336, 0.00023837975459173322, 0.0008907333249226213, 0.0014062307309359312, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.08771783858537674, 0.007821416482329369, 5.662468174705282e-05, 1.4731131869893943e-08, 1.0490747627045494e-05, 0.0011058712843805552, 0.0004291172663215548, 5.523042636923492e-05, 3.919887603842653e-05, 2.7502044304128503e-06, 2.546168616390787e-06, 0.002755208173766732, 0.005722646601498127, 0.0011191918747499585, 0.0015876393299549818, 0.8757357597351074, 0.0010800447780638933, 0.003532156115397811, 0.0015019910642877221, 0.0006949505768716335, 4.3340980482753366e-05, 0.00023435073671862483, 6.934305019967724e-06, 2.7577918615406816e-08, 9.912458835970028e-07, 0.00019156669441144913, 2.9515418646042235e-05, 5.4682345762557816e-06, 2.4805774501146516e-06, 1.4659882481282693e-06, 7.256012395373546e-07, 8.567833720007911e-05, 0.004263886250555515, 0.0007048699189908803, 0.0034579611383378506, 0.0, 0.0, 0.0, 0.0, 0.0], [0.018741462379693985, 0.00013177553773857653, 0.0001151907054008916, 8.336372729900177e-07, 3.8734546592422703e-07, 4.034390804008581e-05, 8.927338058128953e-05, 6.5760018514993135e-06, 6.646589463343844e-05, 2.0145337487065262e-07, 0.00012498503201641142, 0.00022737719700671732, 0.0003596782626118511, 4.087373235961422e-05, 8.65406691445969e-05, 0.00017062197730410844, 0.9720672369003296, 0.004511029925197363, 0.0017741514602676034, 0.00033421185798943043, 0.0006673701573163271, 8.272354534710757e-06, 8.590857760282233e-05, 4.4451999769989925e-07, 1.5395192676237457e-08, 3.916235073120333e-06, 1.856171184044797e-05, 1.744039764162153e-06, 2.306476017110981e-05, 6.561277245964448e-07, 5.5365380831062794e-05, 1.002774115477223e-05, 0.0001100561858038418, 2.098819095408544e-05, 6.0086229495937005e-05, 4.435600567376241e-05, 0.0, 0.0, 0.0, 0.0], [0.021614333614706993, 6.699936784571037e-05, 0.00023382958897855133, 1.946084609016907e-07, 1.0567928256932646e-06, 2.5699444449855946e-05, 0.0002104660088662058, 1.7352494978695177e-06, 6.270136509556323e-06, 8.374804139066327e-08, 2.3547661385237006e-06, 1.8064512914861552e-05, 0.00023376515309792012, 0.001895317924208939, 0.00020409416174516082, 0.0004746883932966739, 0.0029953729826956987, 0.9625540971755981, 0.00691427756100893, 0.00019762477313634008, 0.00011761223140638322, 1.78101454366697e-05, 8.19376073195599e-05, 8.319892685904051e-08, 6.341040403867737e-08, 1.503461476204393e-06, 1.595173125679139e-05, 1.2851933206547983e-07, 7.935282724247372e-07, 1.268652596309039e-07, 5.140130951986066e-07, 3.882859900272706e-08, 5.8853103837464005e-05, 0.0011026777792721987, 9.716644126456231e-05, 8.231291576521471e-05, 0.0007721150759607553, 0.0, 0.0, 0.0], [0.036683231592178345, 5.292119567457121e-06, 1.6796273030195152e-06, 3.2259253601729654e-10, 1.7678923214248243e-08, 9.71175995800877e-06, 6.970066351641435e-06, 3.922034920833539e-06, 1.5272406017174944e-05, 6.423094873753143e-07, 3.0026106401237485e-07, 2.1527788703679107e-05, 8.620285143479123e-07, 2.371107621002011e-05, 1.1333961538184667e-06, 4.370046099211322e-06, 2.1794727217638865e-05, 0.0007244806620292366, 0.9565503597259521, 7.606112922076136e-05, 0.005361164920032024, 3.4096636227332056e-05, 1.4788588487135712e-06, 1.9675812090724776e-09, 6.515209260982147e-09, 7.016106451374071e-07, 1.9081969071521598e-07, 3.605064478051645e-07, 3.97335998059134e-06, 1.5304269709304208e-06, 1.0370927583380762e-07, 7.635036922692962e-07, 1.8985110727953725e-07, 6.1644395827897824e-06, 2.6138322937185876e-06, 1.3529084981200867e-06, 1.0274129635945428e-05, 0.00042383253457956016, 0.0, 0.0], [0.039370130747556686, 0.0003585830272641033, 7.41930998628959e-05, 4.9509686505189165e-05, 4.653078576666303e-06, 4.69761471322272e-05, 0.0002299233601661399, 1.250448349310318e-06, 3.7233094190014526e-05, 8.911907656283802e-08, 7.931947038741782e-05, 0.00013509126438293606, 1.8374801584286615e-05, 0.0001029744089464657, 8.208496728911996e-05, 2.753642911557108e-05, 0.0005808327696286142, 0.001175031648017466, 0.00090598821407184, 0.9545682072639465, 7.417640154017136e-05, 0.0005086685996502638, 0.0001944841060321778, 8.633135439595208e-05, 8.952758889790857e-07, 1.429008443665225e-05, 3.4452030376996845e-05, 2.1258705373838893e-07, 1.695069659035653e-05, 4.816184286937641e-07, 3.1853684049565345e-05, 1.3589491572929546e-05, 2.5087774702114984e-05, 5.98193691985216e-05, 0.00011690238170558587, 1.1342107427481096e-05, 0.0002713192661758512, 0.0002638357982505113, 0.00042731984285637736, 0.0], [0.02540118619799614, 7.441657317031058e-07, 1.083011466107564e-05, 2.249714858848506e-09, 9.22188214680375e-10, 8.20854529592907e-06, 2.5448052838328294e-06, 1.0004332295920904e-07, 3.16087098326534e-05, 2.427833578622085e-07, 5.28918860709382e-07, 1.4262905096984468e-05, 1.5345697192969965e-06, 1.87780551641481e-06, 0.0002028696471825242, 2.141229060725891e-06, 0.00010114459291798994, 0.0009309824672527611, 0.10166086256504059, 5.5514923587907106e-05, 0.8627561926841736, 2.250216311949771e-06, 3.0665501981275156e-05, 5.196882923996782e-09, 3.199170095502524e-11, 8.736409284892943e-08, 1.2241630997777975e-07, 1.2374229418909977e-09, 1.2507468454714399e-05, 4.6357610017366824e-07, 1.629777557354828e-07, 2.719423264352372e-07, 1.1941983757424168e-07, 1.4462457897934655e-07, 5.527898247237317e-05, 3.5639260431707953e-07, 4.93334409839008e-05, 0.00016947872063610703, 0.008456651121377945, 3.8713624235242605e-05]]]}\n",
                            "    )\n",
                            "    </script>"
                        ],
                        "text/plain": [
                            "<circuitsvis.utils.render.RenderedHTML at 0xffff425c9fd0>"
                        ]
                    },
                    "metadata": {
                        "text/html": {
                            "Content-Type": "text/html"
                        }
                    },
                    "output_type": "display_data"
                }
            ],
            "source": [
                "if IN_GITHUB:\n",
                "    torch.manual_seed(50)\n",
                "    \n",
                "induction_head_layer = 5\n",
                "induction_head_index = 5\n",
                "size = (1, 20)\n",
                "input_tensor = torch.randint(1000, 10000, size)\n",
                "\n",
                "single_random_sequence = input_tensor.to(model.cfg.device)\n",
                "repeated_random_sequence = einops.repeat(single_random_sequence, \"batch seq_len -> batch (2 seq_len)\")\n",
                "def visualize_pattern_hook(\n",
                "    pattern: Float[torch.Tensor, \"batch head_index dest_pos source_pos\"],\n",
                "    hook: HookPoint,\n",
                "):\n",
                "    display(\n",
                "        cv.attention.attention_patterns(\n",
                "            tokens=model.to_str_tokens(repeated_random_sequence), \n",
                "            attention=pattern[0, induction_head_index, :, :][None, :, :] # Add a dummy axis, as CircuitsVis expects 3D patterns.\n",
                "        )\n",
                "    )\n",
                "\n",
                "model.run_with_hooks(\n",
                "    repeated_random_sequence, \n",
                "    return_type=None, \n",
                "    fwd_hooks=[(\n",
                "        utils.get_act_name(\"pattern\", induction_head_layer), \n",
                "        visualize_pattern_hook\n",
                "    )]\n",
                ")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Available Models"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "TransformerLens comes with over 40 open source models available, all of which can be loaded into a consistent(-ish) architecture by just changing the name in `from_pretrained`. The open source models available are [documented here](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=jHj79Pj58cgJKdq4t-ygK-4h), and a set of interpretability friendly models I've trained are [documented here](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=NCJ6zH_Okw_mUYAwGnMKsj2m), including a set of toy language models (tiny one to four layer models) and a set of [SoLU models](https://dynalist.io/d/n2ZWtnoYHrU1s4vnFSAQ519J#z=FZ5W6GGcy6OitPEaO733JLqf) up to GPT-2 Medium size (300M parameters). You can see [a table of the official alias and hyper-parameters of available models here](https://github.com/TransformerLensOrg/TransformerLens/blob/main/transformer_lens/model_properties_table.md).\n",
                "\n",
                "**Note:** TransformerLens does not currently support multi-GPU models (which you want for models above eg 7B parameters), but this feature is coming soon!"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "Notably, this means that analysis can be near immediately re-run on a different model by just changing the name - to see this, let's load in DistilGPT-2 (a distilled version of GPT-2, with half as many layers) and copy the code from above to see the induction heads in that model."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 312,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "Using pad_token, but it is not set yet.\n"
                    ]
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Loaded pretrained model distilgpt2 into HookedTransformer\n"
                    ]
                }
            ],
            "source": [
                "# NBVAL_IGNORE_OUTPUT\n",
                "distilgpt2 = HookedTransformer.from_pretrained(\"distilgpt2\", device=device)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 313,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"378fa384-2cc7-4a1c-b871-c643c0e527e8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"378fa384-2cc7-4a1c-b871-c643c0e527e8\")) {                    Plotly.newPlot(                        \"378fa384-2cc7-4a1c-b871-c643c0e527e8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.009922467172145844,0.00014764934894628823,0.011443736031651497,3.4372510526736733e-06,0.0006867930642329156,5.491139290825231e-06,0.008899171836674213,0.0014873683685436845,0.008349093608558178,0.00954477023333311,0.009120927192270756,0.01596890762448311],[0.003026863094419241,0.018020611256361008,0.003568781539797783,0.0006068727816455066,0.013361244462430477,0.002166191814467311,0.005081023555248976,0.01559095922857523,0.0044562919065356255,8.614760008640587e-05,0.003938235342502594,0.014764327555894852],[0.010219043120741844,0.00780068663880229,0.011151125654578209,0.0025892923586070538,0.019908905029296875,0.005215638317167759,0.00943383015692234,0.0015526963397860527,0.017547965049743652,0.011387993581593037,0.021231677383184433,1.451456820402222e-13],[0.007366393692791462,0.22952795028686523,0.8673726916313171,0.016448020935058594,0.01666181907057762,0.011658817529678345,0.01942884363234043,0.20720870792865753,0.014285862445831299,0.016440654173493385,0.9371501803398132,0.49821725487709045],[0.27513042092323303,0.23388457298278809,0.08511187136173248,0.010210886597633362,0.08357562869787216,0.02389742061495781,0.6404961943626404,0.025386787950992584,0.06823859363794327,0.651100218296051,0.015778401866555214,0.06186722218990326],[0.021418118849396706,0.07837661355733871,0.05457017943263054,0.01195995882153511,0.03577948361635208,0.17681372165679932,0.07745278626680374,0.10316655784845352,0.0073539442382752895,0.4502670466899872,0.19098441302776337,0.029486285522580147]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0},\"title\":{\"text\":\"Induction Score by Head in Distil GPT-2\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('378fa384-2cc7-4a1c-b871-c643c0e527e8');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "\n",
                "# We make a tensor to store the induction score for each head. We put it on the model's device to avoid needing to move things between the GPU and CPU, which can be slow.\n",
                "distilgpt2_induction_score_store = torch.zeros((distilgpt2.cfg.n_layers, distilgpt2.cfg.n_heads), device=distilgpt2.cfg.device)\n",
                "def induction_score_hook(\n",
                "    pattern: Float[torch.Tensor, \"batch head_index dest_pos source_pos\"],\n",
                "    hook: HookPoint,\n",
                "):\n",
                "    # We take the diagonal of attention paid from each destination position to source positions seq_len-1 tokens back\n",
                "    # (This only has entries for tokens with index>=seq_len)\n",
                "    induction_stripe = pattern.diagonal(dim1=-2, dim2=-1, offset=1-seq_len)\n",
                "    # Get an average score per head\n",
                "    induction_score = einops.reduce(induction_stripe, \"batch head_index position -> head_index\", \"mean\")\n",
                "    # Store the result.\n",
                "    distilgpt2_induction_score_store[hook.layer(), :] = induction_score\n",
                "\n",
                "# We make a boolean filter on activation names, that's true only on attention pattern names.\n",
                "pattern_hook_names_filter = lambda name: name.endswith(\"pattern\")\n",
                "\n",
                "distilgpt2.run_with_hooks(\n",
                "    repeated_tokens, \n",
                "    return_type=None, # For efficiency, we don't need to calculate the logits\n",
                "    fwd_hooks=[(\n",
                "        pattern_hook_names_filter,\n",
                "        induction_score_hook\n",
                "    )]\n",
                ")\n",
                "\n",
                "imshow(distilgpt2_induction_score_store, xaxis=\"Head\", yaxis=\"Layer\", title=\"Induction Score by Head in Distil GPT-2\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "### An overview of the important open source models in the library\n",
                "\n",
                "* **GPT-2** - the classic generative pre-trained models from OpenAI\n",
                "    * Sizes Small (85M), Medium (300M), Large (700M) and XL (1.5B).\n",
                "    * Trained on ~22B tokens of internet text. ([Open source replication](https://huggingface.co/datasets/openwebtext))\n",
                "* **GPT-Neo** - Eleuther's replication of GPT-2\n",
                "    * Sizes 125M, 1.3B, 2.7B\n",
                "    * Trained on 300B(ish?) tokens of [the Pile](https://pile.eleuther.ai/) a large and diverse dataset including a bunch of code (and weird stuff)\n",
                "* **[OPT](https://ai.facebook.com/blog/democratizing-access-to-large-scale-language-models-with-opt-175b/)** - Meta AI's series of open source models\n",
                "    * Trained on 180B tokens of diverse text.\n",
                "    * 125M, 1.3B, 2.7B, 6.7B, 13B, 30B, 66B\n",
                "* **GPT-J** - Eleuther's 6B parameter model, trained on the Pile\n",
                "* **GPT-NeoX** - Eleuther's 20B parameter model, trained on the Pile\n",
                "* **StableLM** - Stability AI's 3B and 7B models, with and without chat and instruction fine-tuning\n",
                "* **Stanford CRFM models** - a replication of GPT-2 Small and GPT-2 Medium, trained on 5 different random seeds.\n",
                "    * Notably, 600 checkpoints were taken during training per model, and these are available in the library with eg `HookedTransformer.from_pretrained(\"stanford-gpt2-small-a\", checkpoint_index=265)`.\n",
                "- **BERT** - Google's bidirectional encoder-only transformer.\n",
                "    - Size Base (108M), trained on English Wikipedia and BooksCorpus.\n",
                " \n",
                "</details>"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "### An overview of some interpretability-friendly models I've trained and included\n",
                "\n",
                "(Feel free to [reach out](mailto:neelnanda27@gmail.com) if you want more details on any of these models)\n",
                "\n",
                "Each of these models has about ~200 checkpoints taken during training that can also be loaded from TransformerLens, with the `checkpoint_index` argument to `from_pretrained`.\n",
                "\n",
                "Note that all models are trained with a Beginning of Sequence token, and will likely break if given inputs without that! \n",
                "\n",
                "* **Toy Models**: Inspired by [A Mathematical Framework](https://transformer-circuits.pub/2021/framework/index.html), I've trained 12 tiny language models, of 1-4L and each of width 512. I think that interpreting these is likely to be far more tractable than larger models, and both serve as good practice and will likely contain motifs and circuits that generalise to far larger models (like induction heads):\n",
                "    * Attention-Only models (ie without MLPs): attn-only-1l, attn-only-2l, attn-only-3l, attn-only-4l\n",
                "    * GELU models (ie with MLP, and the standard GELU activations): gelu-1l, gelu-2l, gelu-3l, gelu-4l\n",
                "    * SoLU models (ie with MLP, and [Anthropic's SoLU activation](https://transformer-circuits.pub/2022/solu/index.html), designed to make MLP neurons more interpretable): solu-1l, solu-2l, solu-3l, solu-4l\n",
                "    * All models are trained on 22B tokens of data, 80% from C4 (web text) and 20% from Python Code\n",
                "    * Models of the same layer size were trained with the same weight initialization and data shuffle, to more directly compare the effect of different activation functions.\n",
                "* **SoLU** models: A larger scan of models trained with [Anthropic's SoLU activation](https://transformer-circuits.pub/2022/solu/index.html), in the hopes that it makes the MLP neuron interpretability easier. \n",
                "    * A scan up to GPT-2 Medium size, trained on 30B tokens of the same data as toy models, 80% from C4 and 20% from Python code. \n",
                "        * solu-6l (40M), solu-8l (100M), solu-10l (200M), solu-12l (340M)\n",
                "    * An older scan up to GPT-2 Medium size, trained on 15B tokens of [the Pile](https://pile.eleuther.ai/)\n",
                "        * solu-1l-pile (13M), solu-2l-pile (13M), solu-4l-pile (13M), solu-6l-pile (40M), solu-8l-pile (100M), solu-10l-pile (200M), solu-12l-pile (340M)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Other Resources:\n",
                "\n",
                "* [Concrete Steps to Get Started in Mechanistic Interpretability](https://neelnanda.io/getting-started): A guide I wrote for how to get involved in mechanistic interpretability, and how to learn the basic skills\n",
                "* [A Comprehensive Mechanistic Interpretability Explainer](https://neelnanda.io/glossary): An overview of concepts in the field and surrounding ideas in ML and transformers, with long digressions to give context and build intuitions.\n",
                "* [Concrete Open Problems in Mechanistic Interpretability](https://neelnanda.io/concrete-open-problems), a doc I wrote giving a long list of open problems in mechanistic interpretability, and thoughts on how to get started on trying to work on them. \n",
                "    * There's a lot of low-hanging fruit in the field, and I expect that many people reading this could use TransformerLens to usefully make progress on some of these!\n",
                "* Other demos:\n",
                "    * **[Exploratory Analysis Demo](https://neelnanda.io/exploratory-analysis-demo)**, a demonstration of my standard toolkit for how to use TransformerLens to explore a mysterious behaviour in a language model.\n",
                "    * [Interpretability in the Wild](https://github.com/redwoodresearch/Easy-Transformer) a codebase from Arthur Conmy and Alex Variengien at Redwood research using this library to do a detailed and rigorous reverse engineering of the Indirect Object Identification circuit, to accompany their paper\n",
                "        * Note - this was based on an earlier version of this library, called EasyTransformer. It's pretty similar, but several breaking changes have been made since. \n",
                "    * A [recorded walkthrough](https://www.youtube.com/watch?v=yo4QvDn-vsU) of me doing research with TransformerLens on whether a tiny model can re-derive positional information, with [an accompanying Colab](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/main/No_Position_Experiment.ipynb)\n",
                "* [Neuroscope](https://neuroscope.io), a website showing the text in the dataset that most activates each neuron in some selected models. Good to explore to get a sense for what kind of features the model tends to represent, and as a \"wiki\" to get some info\n",
                "    * A tutorial on how to make an [Interactive Neuroscope](https://github.com/TransformerLensOrg/TransformerLens/blob/main/Hacky-Interactive-Lexoscope.ipynb), where you type in text and see the neuron activations over the text update live."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Transformer architecture\n",
                "\n",
                "HookedTransformer is a somewhat adapted GPT-2 architecture, but is computationally identical. The most significant changes are to the internal structure of the attention heads: \n",
                "* The weights (W_K, W_Q, W_V) mapping the residual stream to queries, keys and values are 3 separate matrices, rather than big concatenated one.\n",
                "* The weight matrices (W_K, W_Q, W_V, W_O) and activations (keys, queries, values, z (values mixed by attention pattern)) have separate head_index and d_head axes, rather than flattening them into one big axis.\n",
                "    * The activations all have shape `[batch, position, head_index, d_head]`\n",
                "    * W_K, W_Q, W_V have shape `[head_index, d_model, d_head]` and W_O has shape `[head_index, d_head, d_model]`\n",
                "\n",
                "The actual code is a bit of a mess, as there's a variety of Boolean flags to make it consistent with the various different model families in TransformerLens - to understand it and the internal structure, I instead recommend reading the code in [CleanTransformerDemo](https://colab.research.google.com/github/TransformerLensOrg/TransformerLens/blob/clean-transformer-demo/Clean_Transformer_Demo.ipynb)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Parameter Names\n",
                "\n",
                "Here is a list of the parameters and shapes in the model. By convention, all weight matrices multiply on the right (ie `new_activation = old_activation @ weights + bias`). \n",
                "\n",
                "Reminder of the key hyper-params:\n",
                "* `n_layers`: 12. The number of transformer blocks in the model (a block contains an attention layer and an MLP layer)\n",
                "* `n_heads`: 12. The number of attention heads per attention layer\n",
                "* `d_model`: 768. The residual stream width.\n",
                "* `d_head`: 64. The internal dimension of an attention head activation.\n",
                "* `d_mlp`: 3072. The internal dimension of the MLP layers (ie the number of neurons).\n",
                "* `d_vocab`: 50267. The number of tokens in the vocabulary.\n",
                "* `n_ctx`: 1024. The maximum number of tokens in an input prompt.\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "**Transformer Block parameters:** \n",
                "Replace 0 with the relevant layer index."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 19,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "blocks.0.attn.W_Q torch.Size([12, 768, 64])\n",
                        "blocks.0.attn.W_O torch.Size([12, 64, 768])\n",
                        "blocks.0.attn.b_Q torch.Size([12, 64])\n",
                        "blocks.0.attn.b_O torch.Size([768])\n",
                        "blocks.0.attn.W_K torch.Size([12, 768, 64])\n",
                        "blocks.0.attn.W_V torch.Size([12, 768, 64])\n",
                        "blocks.0.attn.b_K torch.Size([12, 64])\n",
                        "blocks.0.attn.b_V torch.Size([12, 64])\n",
                        "blocks.0.mlp.W_in torch.Size([768, 3072])\n",
                        "blocks.0.mlp.b_in torch.Size([3072])\n",
                        "blocks.0.mlp.W_out torch.Size([3072, 768])\n",
                        "blocks.0.mlp.b_out torch.Size([768])\n"
                    ]
                }
            ],
            "source": [
                "for name, param in model.named_parameters():\n",
                "    if name.startswith(\"blocks.0.\"):\n",
                "        print(name, param.shape)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "**Embedding & Unembedding parameters:**"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 20,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "embed.W_E torch.Size([50257, 768])\n",
                        "pos_embed.W_pos torch.Size([1024, 768])\n",
                        "unembed.W_U torch.Size([768, 50257])\n",
                        "unembed.b_U torch.Size([50257])\n"
                    ]
                }
            ],
            "source": [
                "for name, param in model.named_parameters():\n",
                "    if not name.startswith(\"blocks\"):\n",
                "        print(name, param.shape)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Activation + Hook Names\n",
                "\n",
                "Lets get out a list of the activation/hook names in the model and their shapes. In practice, I recommend using the `utils.get_act_name` function to get the names, but this is a useful fallback, and necessary to eg write a name filter function.\n",
                "\n",
                "Let's do this by entering in a short, 10 token prompt, and add a hook function to each activations to print its name and shape. To avoid spam, let's just add this to activations in the first block or not in a block.\n",
                "\n",
                "Note 1: Each LayerNorm has a hook for the scale factor (ie the standard deviation of the input activations for each token position & batch element) and for the normalized output (ie the input activation with mean 0 and standard deviation 1, but *before* applying scaling or translating with learned weights). LayerNorm is applied every time a layer reads from the residual stream: `ln1` is the LayerNorm before the attention layer in a block, `ln2` the one before the MLP layer, and `ln_final` is the LayerNorm before the unembed. \n",
                "\n",
                "Note 2: *Every* activation apart from the attention pattern and attention scores has shape beginning with `[batch, position]`. The attention pattern and scores have shape `[batch, head_index, dest_position, source_position]` (the numbers are the same, unless we're using caching)."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 316,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Num tokens: 10\n",
                        "hook_embed torch.Size([1, 10, 768])\n",
                        "hook_pos_embed torch.Size([1, 10, 768])\n",
                        "blocks.0.hook_resid_pre torch.Size([1, 10, 768])\n",
                        "blocks.0.ln1.hook_scale torch.Size([1, 10, 1])\n",
                        "blocks.0.ln1.hook_normalized torch.Size([1, 10, 768])\n",
                        "blocks.0.ln1.hook_scale torch.Size([1, 10, 1])\n",
                        "blocks.0.ln1.hook_normalized torch.Size([1, 10, 768])\n",
                        "blocks.0.ln1.hook_scale torch.Size([1, 10, 1])\n",
                        "blocks.0.ln1.hook_normalized torch.Size([1, 10, 768])\n",
                        "blocks.0.attn.hook_q torch.Size([1, 10, 12, 64])\n",
                        "blocks.0.attn.hook_k torch.Size([1, 10, 12, 64])\n",
                        "blocks.0.attn.hook_v torch.Size([1, 10, 12, 64])\n",
                        "blocks.0.attn.hook_attn_scores torch.Size([1, 12, 10, 10])\n",
                        "blocks.0.attn.hook_pattern torch.Size([1, 12, 10, 10])\n",
                        "blocks.0.attn.hook_z torch.Size([1, 10, 12, 64])\n",
                        "blocks.0.hook_attn_out torch.Size([1, 10, 768])\n",
                        "blocks.0.hook_resid_mid torch.Size([1, 10, 768])\n",
                        "blocks.0.ln2.hook_scale torch.Size([1, 10, 1])\n",
                        "blocks.0.ln2.hook_normalized torch.Size([1, 10, 768])\n",
                        "blocks.0.mlp.hook_pre torch.Size([1, 10, 3072])\n",
                        "blocks.0.mlp.hook_post torch.Size([1, 10, 3072])\n",
                        "blocks.0.hook_mlp_out torch.Size([1, 10, 768])\n",
                        "blocks.0.hook_resid_post torch.Size([1, 10, 768])\n",
                        "ln_final.hook_scale torch.Size([1, 10, 1])\n",
                        "ln_final.hook_normalized torch.Size([1, 10, 768])\n"
                    ]
                }
            ],
            "source": [
                "test_prompt = \"The quick brown fox jumped over the lazy dog\"\n",
                "print(\"Num tokens:\", len(model.to_tokens(test_prompt)[0]))\n",
                "\n",
                "def print_name_shape_hook_function(activation, hook):\n",
                "    print(hook.name, activation.shape)\n",
                "\n",
                "not_in_late_block_filter = lambda name: name.startswith(\"blocks.0.\") or not name.startswith(\"blocks\")\n",
                "\n",
                "model.run_with_hooks(\n",
                "    test_prompt,\n",
                "    return_type=None,\n",
                "    fwd_hooks=[(not_in_late_block_filter, print_name_shape_hook_function)],\n",
                ")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Folding LayerNorm (For the Curious)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "(For the curious - this is an important technical detail that's worth understanding, especially if you have preconceptions about how transformers work, but not necessary to use TransformerLens)\n",
                "\n",
                "LayerNorm is a normalization technique used by transformers, analogous to BatchNorm but more friendly to massive parallelisation. No one *really* knows why it works, but it seems to improve model numerical stability. Unlike BatchNorm, LayerNorm actually changes the functional form of the model, which makes it a massive pain for interpretability! \n",
                "\n",
                "Folding LayerNorm is a technique to make it lower overhead to deal with, and the flags `center_writing_weights` and `fold_ln` in `HookedTransformer.from_pretrained` apply this automatically (they default to True). These simplify the internal structure without changing the weights.\n",
                "\n",
                "Intuitively, LayerNorm acts on each residual stream vector (ie for each batch element and token position) independently, sets their mean to 0 (centering) and standard deviation to 1 (normalizing) (*across* the residual stream dimension - very weird!), and then applies a learned elementwise scaling and translation to each vector.\n",
                "\n",
                "Mathematically, centering is a linear map, normalizing is *not* a linear map, and scaling and translation are linear maps. \n",
                "* **Centering:** LayerNorm is applied every time a layer reads from the residual stream, so the mean of any residual stream vector can never matter - `center_writing_weights` set every weight matrix writing to the residual to have zero mean. \n",
                "* **Normalizing:** Normalizing is not a linear map, and cannot be factored out. The `hook_scale` hook point lets you access and control for this.\n",
                "* **Scaling and Translation:** Scaling and translation are linear maps, and are always followed by another linear map. The composition of two linear maps is another linear map, so we can *fold* the scaling and translation weights into the weights of the subsequent layer, and simplify things without changing the underlying computation. \n",
                "\n",
                "[See the docs for more details](https://github.com/TransformerLensOrg/TransformerLens/blob/main/further_comments.md#what-is-layernorm-folding-fold_ln)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "A fun consequence of LayerNorm folding is that it creates a bias across the unembed, a `d_vocab` length vector that is added to the output logits - GPT-2 is not trained with this, but it *is* trained with a final LayerNorm that contains a bias. \n",
                "\n",
                "Turns out, this LayerNorm bias learns structure of the data that we can only see after folding! In particular, it essentially learns **unigram statistics** - rare tokens get suppressed, common tokens get boosted, by pretty dramatic degrees! Let's list the top and bottom 20 - at the top we see common punctuation and words like \" the\" and \" and\", at the bottom we see weird-ass tokens like \" RandomRedditor\":"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 317,
            "metadata": {},
            "outputs": [],
            "source": [
                "unembed_bias = model.unembed.b_U\n",
                "bias_values, bias_indices = unembed_bias.sort(descending=True)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 318,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Top 20 values\n",
                        "7.03 ','\n",
                        "6.98 ' the'\n",
                        "6.68 ' and'\n",
                        "6.49 '.'\n",
                        "6.48 '\\n'\n",
                        "6.47 ' a'\n",
                        "6.41 ' in'\n",
                        "6.25 ' to'\n",
                        "6.16 ' of'\n",
                        "6.04 '-'\n",
                        "6.03 ' ('\n",
                        "5.88 ' \"'\n",
                        "5.80 ' for'\n",
                        "5.72 ' that'\n",
                        "5.64 ' on'\n",
                        "5.59 ' is'\n",
                        "5.52 ' as'\n",
                        "5.49 ' at'\n",
                        "5.45 ' with'\n",
                        "5.44 ' or'\n",
                        "...\n",
                        "Bottom 20 values\n",
                        "-3.82 ' サーティ'\n",
                        "-3.83 '\\x18'\n",
                        "-3.83 '\\x14'\n",
                        "-3.83 ' RandomRedditor'\n",
                        "-3.83 '龍�'\n",
                        "-3.83 '�'\n",
                        "-3.83 '\\x1b'\n",
                        "-3.83 '�'\n",
                        "-3.83 '\\x05'\n",
                        "-3.83 '\\x00'\n",
                        "-3.83 '\\x06'\n",
                        "-3.83 '\\x07'\n",
                        "-3.83 '\\x0c'\n",
                        "-3.83 '\\x02'\n",
                        "-3.83 'oreAndOnline'\n",
                        "-3.84 '\\x11'\n",
                        "-3.84 '�'\n",
                        "-3.84 '\\x10'\n",
                        "-3.84 '�'\n",
                        "-3.84 '�'\n"
                    ]
                }
            ],
            "source": [
                "top_k = 20\n",
                "print(f\"Top {top_k} values\")\n",
                "for i in range(top_k):\n",
                "    print(f\"{bias_values[i].item():.2f} {repr(model.to_string(bias_indices[i]))}\")\n",
                "\n",
                "print(\"...\")\n",
                "print(f\"Bottom {top_k} values\")\n",
                "for i in range(top_k, 0, -1):\n",
                "    print(f\"{bias_values[-i].item():.2f} {repr(model.to_string(bias_indices[-i]))}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "This can have real consequences for interpretability - for example, this bias favours \" John\" over \" Mary\" by about 1.2, about 1/3 of the effect size of the Indirect Object Identification Circuit! All other things being the same, this makes the John token 3.6x times more likely than the Mary token."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 319,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "John bias: 2.8995\n",
                        "Mary bias: 1.6034\n",
                        "Prob ratio bias: 3.6550x\n"
                    ]
                }
            ],
            "source": [
                "john_bias = model.unembed.b_U[model.to_single_token(' John')]\n",
                "mary_bias = model.unembed.b_U[model.to_single_token(' Mary')]\n",
                "\n",
                "print(f\"John bias: {john_bias.item():.4f}\")\n",
                "print(f\"Mary bias: {mary_bias.item():.4f}\")\n",
                "print(f\"Prob ratio bias: {torch.exp(john_bias - mary_bias).item():.4f}x\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Features\n",
                "\n",
                "An overview of some other important features of the library. I recommend checking out the [Exploratory Analysis Demo](https://colab.research.google.com/github/TransformerLensOrg/Easy-Transformer/blob/main/Exploratory_Analysis_Demo.ipynb) for some other important features not mentioned here, and for a demo of what using the library in practice looks like."
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Dealing with tokens\n",
                "\n",
                "**Tokenization** is one of the most annoying features of studying language models. We want language models to be able to take in arbitrary text as input, but the transformer architecture needs the inputs to be elements of a fixed, finite vocabulary. The solution to this is **tokens**, a fixed vocabulary of \"sub-words\", that any natural language can be broken down into with a **tokenizer**. This is invertible, and we can recover the original text, called **de-tokenization**. \n",
                "\n",
                "TransformerLens comes with a range of utility functions to deal with tokenization. Different models can have different tokenizers, so these are all methods on the model.\n",
                "\n",
                "get_token_position, to_tokens, to_string, to_str_tokens, prepend_bos, to_single_token"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The first thing you need to figure out is *how* things are tokenized. `model.to_str_tokens` splits a string into the tokens *as a list of substrings*, and so lets you explore what the text looks like. To demonstrate this, let's use it on this paragraph.\n",
                "\n",
                "Some observations - there are a lot of arbitrary-ish details in here!\n",
                "* The tokenizer splits on spaces, so no token contains two words.\n",
                "* Tokens include the preceding space, and whether the first token is a capital letter. `how` and ` how` are different tokens!\n",
                "* Common words are single tokens, even if fairly long (` paragraph`) while uncommon words are split into multiple tokens (` token|ized`).\n",
                "* Tokens *mostly* split on punctuation characters (eg `*` and `.`), but eg `'s` is a single token."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 320,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "['<|endoftext|>', 'The', ' first', ' thing', ' you', ' need', ' to', ' figure', ' out', ' is', ' *', 'how', '*', ' things', ' are', ' token', 'ized', '.', ' `', 'model', '.', 'to', '_', 'str', '_', 't', 'ok', 'ens', '`', ' splits', ' a', ' string', ' into', ' the', ' tokens', ' *', 'as', ' a', ' list', ' of', ' sub', 'strings', '*,', ' and', ' so', ' lets', ' you', ' explore', ' what', ' the', ' text', ' looks', ' like', '.', ' To', ' demonstrate', ' this', ',', ' let', \"'s\", ' use', ' it', ' on', ' this', ' paragraph', '.']\n"
                    ]
                }
            ],
            "source": [
                "example_text = \"The first thing you need to figure out is *how* things are tokenized. `model.to_str_tokens` splits a string into the tokens *as a list of substrings*, and so lets you explore what the text looks like. To demonstrate this, let's use it on this paragraph.\"\n",
                "example_text_str_tokens = model.to_str_tokens(example_text)\n",
                "print(example_text_str_tokens)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The transformer needs to take in a sequence of integers, not strings, so we need to convert these tokens into integers. `model.to_tokens` does this, and returns a tensor of integers on the model's device (shape `[batch, position]`). It maps a string to a batch of size 1."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 321,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "tensor([[50256,   464,   717,  1517,   345,   761,   284,  3785,   503,   318,\n",
                        "          1635,  4919,     9,  1243,   389, 11241,  1143,    13,  4600, 19849,\n",
                        "            13,  1462,    62,  2536,    62,    83,   482,   641,    63, 30778,\n",
                        "           257,  4731,   656,   262, 16326,  1635,   292,   257,  1351,   286,\n",
                        "           850, 37336, 25666,   290,   523,  8781,   345,  7301,   644,   262,\n",
                        "          2420,  3073,   588,    13,  1675, 10176,   428,    11,  1309,   338,\n",
                        "           779,   340,   319,   428,  7322,    13]])\n"
                    ]
                }
            ],
            "source": [
                "example_text_tokens = model.to_tokens(example_text)\n",
                "print(example_text_tokens)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "`to_tokens` can also take in a list of strings, and return a batch of size `len(strings)`. If the strings are different numbers of tokens, it adds a PAD token to the end of the shorter strings to make them the same length.\n",
                "\n",
                "(Note: In GPT-2, 50256 signifies both the beginning of sequence, end of sequence and padding token - see the `prepend_bos` section for details)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 322,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "tensor([[50256,   464,  3797,  3332,   319,   262,  2603,    13, 50256, 50256],\n",
                        "        [50256,   464,  3797,  3332,   319,   262,  2603,  1107,  1327,    13]])\n"
                    ]
                }
            ],
            "source": [
                "example_multi_text = [\"The cat sat on the mat.\", \"The cat sat on the mat really hard.\"]\n",
                "example_multi_text_tokens = model.to_tokens(example_multi_text)\n",
                "print(example_multi_text_tokens)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "`model.to_single_token` is a convenience function that takes in a string corresponding to a *single* token and returns the corresponding integer. This is useful for eg looking up the logit corresponding to a single token. \n",
                "\n",
                "For example, let's input `The cat sat on the mat.` to GPT-2, and look at the log prob predicting that the next token is ` The`. \n",
                "\n",
                "<details><summary>Technical notes</summary>\n",
                "\n",
                "Note that if we input a string to the model, it's implicitly converted to a string with `to_tokens`. \n",
                "\n",
                "Note further that the log probs have shape `[batch, position, d_vocab]==[1, 8, 50257]`, with a vector of log probs predicting the next token for *every* token position. GPT-2 uses causal attention which means heads can only look backwards (equivalently, information can only move forwards in the model.), so the log probs at position k are only a function of the first k tokens, and it can't just cheat and look at the k+1 th token. This structure lets it generate text more efficiently, and lets it treat every *token* as a training example, rather than every *sequence*.\n",
                "</details>"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 323,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Probability tensor shape [batch, position, d_vocab] == torch.Size([1, 8, 50257])\n",
                        "| The| probability: 11.98%\n"
                    ]
                }
            ],
            "source": [
                "cat_text = \"The cat sat on the mat.\"\n",
                "cat_logits = model(cat_text)\n",
                "cat_probs = cat_logits.softmax(dim=-1)\n",
                "print(f\"Probability tensor shape [batch, position, d_vocab] == {cat_probs.shape}\")\n",
                "\n",
                "capital_the_token_index = model.to_single_token(\" The\")\n",
                "print(f\"| The| probability: {cat_probs[0, -1, capital_the_token_index].item():.2%}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "`model.to_string` is the inverse of `to_tokens` and maps a tensor of integers to a string or list of strings. It also works on integers and lists of integers.\n",
                "\n",
                "For example, let's look up token 256 (due to technical details of tokenization, this will be the most common pair of ASCII characters!), and also verify that our tokens above map back to a string."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 324,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Token 256 - the most common pair of ASCII characters: | t|\n",
                        "De-Tokenizing the example tokens: <|endoftext|>The first thing you need to figure out is *how* things are tokenized. `model.to_str_tokens` splits a string into the tokens *as a list of substrings*, and so lets you explore what the text looks like. To demonstrate this, let's use it on this paragraph.\n"
                    ]
                }
            ],
            "source": [
                "print(f\"Token 256 - the most common pair of ASCII characters: |{model.to_string(256)}|\")\n",
                "# Squeeze means to remove dimensions of length 1. \n",
                "# Here, that removes the dummy batch dimension so it's a rank 1 tensor and returns a string\n",
                "# Rank 2 tensors map to a list of strings\n",
                "print(f\"De-Tokenizing the example tokens: {model.to_string(example_text_tokens.squeeze())}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "A related annoyance of tokenization is that it's hard to figure out how many tokens a string will break into. `model.get_token_position(single_token, tokens)` returns the position of `single_token` in `tokens`. `tokens` can be either a string or a tensor of tokens. \n",
                "\n",
                "Note that position is zero-indexed, it's two (ie third) because there's a beginning of sequence token automatically prepended (see the next section for details)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 325,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "With BOS: 2\n",
                        "Without BOS: 1\n"
                    ]
                }
            ],
            "source": [
                "print(\"With BOS:\", model.get_token_position(\" cat\", \"The cat sat on the mat\"))\n",
                "print(\"Without BOS:\", model.get_token_position(\" cat\", \"The cat sat on the mat\", prepend_bos=False))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "If there are multiple copies of the token, we can set `mode=\"first\"` to find the first occurence's position and `mode=\"last\"` to find the last"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 326,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "First occurence 2\n",
                        "Final occurence 13\n"
                    ]
                }
            ],
            "source": [
                "print(\"First occurence\", model.get_token_position(\n",
                "    \" cat\", \n",
                "    \"The cat sat on the mat. The mat sat on the cat.\", \n",
                "    mode=\"first\"))\n",
                "print(\"Final occurence\", model.get_token_position(\n",
                "    \" cat\", \n",
                "    \"The cat sat on the mat. The mat sat on the cat.\", \n",
                "    mode=\"last\"))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "In general, tokenization is a pain, and full of gotchas. I highly recommend just playing around with different inputs and their tokenization and getting a feel for it. As another \"fun\" example, let's look at the tokenization of arithmetic expressions - tokens do *not* contain consistent numbers of digits. (This makes it even more impressive that GPT-3 can do arithmetic!)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 327,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "['<|endoftext|>', '23', '42', '+', '2017', '=', '214', '45']\n",
                        "['<|endoftext|>', '1000', '+', '1', '000000', '=', '9999', '99']\n"
                    ]
                }
            ],
            "source": [
                "print(model.to_str_tokens(\"2342+2017=21445\"))\n",
                "print(model.to_str_tokens(\"1000+1000000=999999\"))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "I also *highly* recommend investigating prompts with easy tokenization when starting out - ideally key words should form a single token, be in the same position in different prompts, have the same total length, etc. Eg study Indirect Object Identification with common English names like ` Tim` rather than ` Ne|el`. Transformers need to spend some parameters in early layers converting multi-token words to a single feature, and then de-converting this in the late layers, and unless this is what you're explicitly investigating, this will make the behaviour you're investigating be messier."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Gotcha: `prepend_bos`\n",
                "\n",
                "Key Takeaway: **If you get weird off-by-one errors, check whether there's an unexpected `prepend_bos`!**"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "A weirdness you may have noticed in the above is that `to_tokens` and `to_str_tokens` added a weird `<|endoftext|>` to the start of each prompt. TransformerLens does this by default, and it can easily trip up new users. Notably, **this includes `model.forward`** (which is what's implicitly used when you do eg `model(\"Hello World\")`). This is called a **Beginning of Sequence (BOS)** token, and it's a special token used to mark the beginning of the sequence. Confusingly, in GPT-2, the End of Sequence (EOS), Beginning of Sequence (BOS) and Padding (PAD) tokens are all the same, `<|endoftext|>` with index `50256`.\n",
                "\n",
                "**Gotcha:** You only want to prepend a BOS token at the *start* of a prompt. If you, eg, want to input a question followed by an answer, and want to tokenize these separately, you do *not* want to prepend_bos on the answer."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 328,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Logits shape by default (with BOS) torch.Size([1, 3, 50257])\n",
                        "Logits shape with BOS torch.Size([1, 3, 50257])\n",
                        "Logits shape without BOS - only 2 positions! torch.Size([1, 2, 50257])\n"
                    ]
                }
            ],
            "source": [
                "print(\"Logits shape by default (with BOS)\", model(\"Hello World\").shape)\n",
                "print(\"Logits shape with BOS\", model(\"Hello World\", prepend_bos=True).shape)\n",
                "print(\"Logits shape without BOS - only 2 positions!\", model(\"Hello World\", prepend_bos=False).shape)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "`prepend_bos` is a bit of a hack, and I've gone back and forth on what the correct default here is. The reason I do this is that transformers tend to treat the first token weirdly - this doesn't really matter in training (where all inputs are >1000 tokens), but this can be a big issue when investigating short prompts! The reason for this is that attention patterns are a probability distribution and so need to add up to one, so to simulate being \"off\" they normally look at the first token. Giving them a BOS token lets the heads rest by looking at that, preserving the information in the first \"real\" token.\n",
                "\n",
                "Further, *some* models are trained to need a BOS token (OPT and my interpretability-friendly models are, GPT-2 and GPT-Neo are not). But despite GPT-2 not being trained with this, empirically it seems to make interpretability easier.\n",
                "\n",
                "(However, if you want to change the default behaviour to *not* prepending a BOS token, pass `default_prepend_bos=False` when you instantiate the model, e.g., `model = HookedTransformer.from_pretrained('gpt2', default_prepend_bos=False)`.)\n",
                "\n",
                "For example, the model can get much worse at Indirect Object Identification without a BOS (and with a name as the first token):"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 329,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Logit difference with BOS: 6.754\n",
                        "Logit difference without BOS: 2.782\n"
                    ]
                }
            ],
            "source": [
                "ioi_logits_with_bos = model(\"Claire and Mary went to the shops, then Mary gave a bottle of milk to\", prepend_bos=True)\n",
                "mary_logit_with_bos = ioi_logits_with_bos[0, -1, model.to_single_token(\" Mary\")].item()\n",
                "claire_logit_with_bos = ioi_logits_with_bos[0, -1, model.to_single_token(\" Claire\")].item()\n",
                "print(f\"Logit difference with BOS: {(claire_logit_with_bos - mary_logit_with_bos):.3f}\")\n",
                "\n",
                "ioi_logits_without_bos = model(\"Claire and Mary went to the shops, then Mary gave a bottle of milk to\", prepend_bos=False)\n",
                "mary_logit_without_bos = ioi_logits_without_bos[0, -1, model.to_single_token(\" Mary\")].item()\n",
                "claire_logit_without_bos = ioi_logits_without_bos[0, -1, model.to_single_token(\" Claire\")].item()\n",
                "print(f\"Logit difference without BOS: {(claire_logit_without_bos - mary_logit_without_bos):.3f}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Though, note that this also illustrates another gotcha - when `Claire` is at the start of a sentence (no preceding space), it's actually *two* tokens, not one, which probably confuses the relevant circuit. (Note - in this test we put `prepend_bos=False`, because we want to analyse the tokenization of a specific string, not to give an input to the model!)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 330,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "| Claire| -> [' Claire']\n",
                        "|Claire| -> ['Cl', 'aire']\n"
                    ]
                }
            ],
            "source": [
                "print(f\"| Claire| -> {model.to_str_tokens(' Claire', prepend_bos=False)}\")\n",
                "print(f\"|Claire| -> {model.to_str_tokens('Claire', prepend_bos=False)}\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Factored Matrix Class\n",
                "\n",
                "In transformer interpretability, we often need to analyse low rank factorized matrices - a matrix $M = AB$, where M is `[large, large]`, but A is `[large, small]` and B is `[small, large]`. This is a common structure in transformers, and the `FactoredMatrix` class is a convenient way to work with these. It implements efficient algorithms for various operations on these, such as computing the trace, eigenvalues, Frobenius norm, singular value decomposition, and products with other matrices. It can (approximately) act as a drop-in replacement for the original matrix, and supports leading batch dimensions to the factored matrix. \n",
                "\n",
                "<details><summary>Why are low-rank factorized matrices useful for transformer interpretability?</summary>\n",
                "\n",
                "As argued in [A Mathematical Framework](https://transformer-circuits.pub/2021/framework/index.html), an unexpected fact about transformer attention heads is that rather than being best understood as keys, queries and values (and the requisite weight matrices), they're actually best understood as two low rank factorized matrices. \n",
                "* **Where to move information from:** $W_QK = W_Q W_K^T$, used for determining the attention pattern - what source positions to move information from and what destination positions to move them to.\n",
                "    * Intuitively, residual stream -> query and residual stream -> key are linear maps, *and* `attention_score = query @ key.T` is a linear map, so the whole thing can be factored into one big bilinear form `residual @ W_QK @ residual.T`\n",
                "* **What information to move:** $W_OV = W_V W_O$, used to determine what information to copy from the source position to the destination position (weighted by the attention pattern weight from that destination to that source). \n",
                "    * Intuitively, the residual stream is a `[position, d_model]` tensor (ignoring batch). The attention pattern acts on the *position* dimension (where to move information from and to) and the value and output weights act on the *d_model* dimension - ie *what* information is contained at that source position. So we can factor it all into `attention_pattern @ residual @ W_V @ W_O`, and so only need to care about `W_OV = W_V @ W_O`\n",
                "* Note - the internal head dimension is smaller than the residual stream dimension, so the factorization is low rank. (here, `d_model=768` and `d_head=64`)\n",
                "</details>"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Basic Examples"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can use the basic class directly - let's make a factored matrix directly and look at the basic operations:"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 331,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Norms:\n",
                        "tensor(9.9105)\n",
                        "tensor(9.9105)\n",
                        "Right dimension: 5, Left dimension: 5, Hidden dimension: 2\n"
                    ]
                }
            ],
            "source": [
                "if IN_GITHUB:\n",
                "    torch.manual_seed(50)\n",
                "A = torch.randn(5, 2)\n",
                "B = torch.randn(2, 5)\n",
                "\n",
                "AB = A @ B\n",
                "AB_factor = FactoredMatrix(A, B)\n",
                "print(\"Norms:\")\n",
                "print(AB.norm())\n",
                "print(AB_factor.norm())\n",
                "\n",
                "print(f\"Right dimension: {AB_factor.rdim}, Left dimension: {AB_factor.ldim}, Hidden dimension: {AB_factor.mdim}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can also look at the eigenvalues and singular values of the matrix. Note that, because the matrix is rank 2 but 5 by 5, the final 3 eigenvalues and singular values are zero - the factored class omits the zeros."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 332,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Eigenvalues:\n",
                        "tensor([-6.2877e+00+0.j,  1.9337e-07+0.j,  2.3121e+00+0.j, -5.9987e-07+0.j,\n",
                        "        -1.1409e-07+0.j])\n",
                        "tensor([-6.2877+0.j,  2.3121+0.j])\n",
                        "\n",
                        "Singular Values:\n",
                        "tensor([8.3126e+00, 5.3963e+00, 1.4519e-07, 7.4293e-08, 2.1726e-09])\n",
                        "tensor([8.3126, 5.3963])\n"
                    ]
                }
            ],
            "source": [
                "# NBVAL_IGNORE_OUTPUT\n",
                "print(\"Eigenvalues:\")\n",
                "print(torch.linalg.eig(AB).eigenvalues)\n",
                "print(AB_factor.eigenvalues)\n",
                "print()\n",
                "print(\"Singular Values:\")\n",
                "print(torch.linalg.svd(AB).S)\n",
                "print(AB_factor.S)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can multiply with other matrices - it automatically chooses the smallest possible dimension to factor along (here it's 2, rather than 5)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 333,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Unfactored: torch.Size([5, 300]) tensor(160.0830)\n",
                        "Factored: torch.Size([5, 300]) tensor(160.0830)\n",
                        "Right dimension: 300, Left dimension: 5, Hidden dimension: 2\n"
                    ]
                }
            ],
            "source": [
                "if IN_GITHUB:\n",
                "    torch.manual_seed(50)\n",
                "    \n",
                "C = torch.randn(5, 300)\n",
                "\n",
                "ABC = AB @ C\n",
                "ABC_factor = AB_factor @ C\n",
                "print(\"Unfactored:\", ABC.shape, ABC.norm().round(decimals=3))\n",
                "print(\"Factored:\", ABC_factor.shape, ABC_factor.norm().round(decimals=3))\n",
                "print(f\"Right dimension: {ABC_factor.rdim}, Left dimension: {ABC_factor.ldim}, Hidden dimension: {ABC_factor.mdim}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "If we want to collapse this back to an unfactored matrix, we can use the AB property to get the product:"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 334,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "tensor(True)\n"
                    ]
                }
            ],
            "source": [
                "AB_unfactored = AB_factor.AB\n",
                "print(torch.isclose(AB_unfactored, AB).all())"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Medium Example: Eigenvalue Copying Scores\n",
                "\n",
                "(This is a more involved example of how to use the factored matrix class, skip it if you aren't following)\n",
                "\n",
                "For a more involved example, let's look at the eigenvalue copying score from [A Mathematical Framework](https://transformer-circuits.pub/2021/framework/index.html) of the OV circuit for various heads. The OV Circuit for a head (the factorised matrix $W_OV = W_V W_O$) is a linear map that determines what information is moved from the source position to the destination position. Because this is low rank, it can be thought of as *reading in* some low rank subspace of the source residual stream and *writing to* some low rank subspace of the destination residual stream (with maybe some processing happening in the middle).\n",
                "\n",
                "A common operation for this will just be to *copy*, ie to have the same reading and writing subspace, and to do minimal processing in the middle. Empirically, this tends to coincide with the OV Circuit having (approximately) positive real eigenvalues. I mostly assert this as an empirical fact, but intuitively, operations that involve mapping eigenvectors to different directions (eg rotations) tend to have complex eigenvalues. And operations that preserve eigenvector direction but negate it tend to have negative real eigenvalues. And \"what happens to the eigenvectors\" is a decent proxy for what happens to an arbitrary vector.\n",
                "\n",
                "We can get a score for \"how positive real the OV circuit eigenvalues are\" with $\\frac{\\sum \\lambda_i}{\\sum |\\lambda_i|}$, where $\\lambda_i$ are the eigenvalues of the OV circuit. This is a bit of a hack, but it seems to work well in practice."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Let's use FactoredMatrix to compute this for every head in the model! We use the helper `model.OV` to get the concatenated OV circuits for all heads across all layers in the model. This has the shape `[n_layers, n_heads, d_model, d_model]`, where `n_layers` and `n_heads` are batch dimensions and the final two dimensions are factorised as `[n_layers, n_heads, d_model, d_head]` and `[n_layers, n_heads, d_head, d_model]` matrices.\n",
                "\n",
                "We can then get the eigenvalues for this, where there are separate eigenvalues for each element of the batch (a `[n_layers, n_heads, d_head]` tensor of complex numbers), and calculate the copying score."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 335,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "FactoredMatrix: Shape(torch.Size([12, 12, 768, 768])), Hidden Dim(64)\n"
                    ]
                }
            ],
            "source": [
                "OV_circuit_all_heads = model.OV\n",
                "print(OV_circuit_all_heads)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 336,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "torch.Size([12, 12, 64])\n",
                        "torch.complex64\n"
                    ]
                }
            ],
            "source": [
                "OV_circuit_all_heads_eigenvalues = OV_circuit_all_heads.eigenvalues \n",
                "print(OV_circuit_all_heads_eigenvalues.shape)\n",
                "print(OV_circuit_all_heads_eigenvalues.dtype)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 337,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"a09834af-1357-479f-b958-ca36d205cbf8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a09834af-1357-479f-b958-ca36d205cbf8\")) {                    Plotly.newPlot(                        \"a09834af-1357-479f-b958-ca36d205cbf8\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.7775010466575623,0.3527269959449768,0.25961846113204956,0.6670257449150085,0.8384260535240173,0.5584430694580078,0.8444744944572449,0.4137910008430481,0.24488940834999084,0.028157662600278854,0.3584098219871521,0.16288265585899353],[-0.45419126749038696,-0.6529328227043152,-0.5484569072723389,-0.7990369200706482,-0.7736425995826721,-0.8522581458091736,0.9774324893951416,0.6626249551773071,-0.7303224205970764,-0.7007019519805908,-0.6946625709533691,-0.9996722340583801],[-0.7837163805961609,0.8967759013175964,0.4750954806804657,-0.667197585105896,0.7881461977958679,-0.8547751307487488,-0.9054184556007385,-0.5749384760856628,-0.32175111770629883,-0.028594352304935455,-0.9247617721557617,-0.9699268341064453],[0.5864037275314331,-0.7614347338676453,0.5971695780754089,0.7854393720626831,-0.8788883686065674,0.3908745050430298,0.044738516211509705,0.11028008162975311,-0.8169988989830017,0.22129566967487335,-0.9939578771591187,0.5774399042129517],[0.525479257106781,0.3049013912677765,-0.10729152709245682,0.9433151483535767,-0.9314428567886353,0.5273632407188416,-0.4264712631702423,-0.9984429478645325,0.5296756029129028,0.8604294061660767,-0.8895052075386047,0.9556970596313477],[0.6629186868667603,0.42956963181495667,0.9736858010292053,0.6555483937263489,0.12201889604330063,0.7442770004272461,0.5037952661514282,0.9525359272956848,-0.6507164239883423,-0.9316279292106628,0.9791510105133057,-0.9972584843635559],[0.9613031148910522,0.7501779794692993,-0.3806658685207367,0.6429786682128906,0.9557769298553467,-0.9428839683532715,-0.9948079586029053,0.785298764705658,0.9657301306724548,0.707301676273346,0.3687230050563812,0.8128011226654053],[0.9659481644630432,0.9730121493339539,0.31900617480278015,-0.30290520191192627,0.9790953397750854,0.9357923269271851,-0.5550313591957092,-0.005466493312269449,0.9867776036262512,0.8249565958976746,0.566429615020752,0.1000526174902916],[-0.9464486837387085,-0.25471997261047363,0.6522327661514282,0.1415255218744278,0.9884140491485596,0.9860583543777466,0.6949270367622375,0.9901810884475708,0.9791202545166016,-0.2359553426504135,-0.9820711612701416,0.6506689190864563],[0.9895943999290466,-0.29178157448768616,0.9714024662971497,0.9951602220535278,0.18783769011497498,-0.9460937976837158,0.47801902890205383,-0.2489192932844162,0.9437097907066345,0.11866245418787003,0.9941242933273315,-0.38088178634643555],[0.9564487934112549,0.5542725920677185,0.42118048667907715,0.6628789901733398,0.8659590482711792,0.9937117695808411,0.9069075584411621,0.39811065793037415,-0.4134220480918884,0.9971913695335388,0.34596705436706543,0.9938657283782959],[0.5891268849372864,0.9313740134239197,0.9268401861190796,0.9993564486503601,0.6227539777755737,0.8463947772979736,0.6584346294403076,0.8423126339912415,0.2978496253490448,0.8728679418563843,0.9963144659996033,0.986752450466156]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('a09834af-1357-479f-b958-ca36d205cbf8');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "OV_copying_score = OV_circuit_all_heads_eigenvalues.sum(dim=-1).real / OV_circuit_all_heads_eigenvalues.abs().sum(dim=-1)\n",
                "imshow(utils.to_numpy(OV_copying_score), xaxis=\"Head\", yaxis=\"Layer\", title=\"OV Copying Score for each head in GPT-2 Small\", zmax=1.0, zmin=-1.0)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Head 11 in Layer 11 (L11H11) has a high copying score, and if we plot the eigenvalues they look approximately as expected."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 338,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"1e3711f8-c7bc-42a1-aea4-6c909a93f24e\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1e3711f8-c7bc-42a1-aea4-6c909a93f24e\")) {                    Plotly.newPlot(                        \"1e3711f8-c7bc-42a1-aea4-6c909a93f24e\",                        [{\"hovertemplate\":\"Real=%{x}\\u003cbr\\u003eImaginary=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[-2.1397297382354736,1.4152636528015137,3.444455146789551,4.027669906616211,8.882655143737793,4.866769790649414,4.866769790649414,4.843714714050293,4.843714714050293,8.477535247802734,8.216809272766113,8.216809272766113,5.07860803604126,7.855461120605469,7.855461120605469,5.365756034851074,5.365756034851074,5.563426971435547,5.563426971435547,5.4217329025268555,7.769133567810059,7.769133567810059,7.0422773361206055,7.0422773361206055,5.675145626068115,5.675145626068115,7.678577423095703,7.678577423095703,6.573329925537109,6.573329925537109,7.67294979095459,7.17219877243042,7.17219877243042,7.423620223999023,7.423620223999023,7.470810413360596,6.089095115661621,6.089095115661621,6.306834697723389,6.306834697723389,6.511750221252441,6.511750221252441,5.955246448516846,5.955246448516846,5.858811378479004,5.858811378479004,7.147887229919434,7.147887229919434,7.185712814331055,7.185712814331055,6.670608043670654,6.670608043670654,6.735983848571777,6.735983848571777,6.149757385253906,6.149757385253906,6.288776874542236,6.288776874542236,6.344796657562256,6.625571250915527,6.625571250915527,6.8991899490356445,6.8991899490356445,6.85640811920166],\"xaxis\":\"x\",\"y\":[0.0,0.0,0.0,0.0,0.0,0.4185231328010559,-0.4185231328010559,0.09079001098871231,-0.09079001098871231,0.0,0.40868881344795227,-0.40868881344795227,0.0,0.7007214426994324,-0.7007214426994324,0.46421411633491516,-0.46421411633491516,0.5558239817619324,-0.5558239817619324,0.0,0.4705662131309509,-0.4705662131309509,1.0298681259155273,-1.0298681259155273,0.48253530263900757,-0.48253530263900757,0.3356489837169647,-0.3356489837169647,0.9988697171211243,-0.9988697171211243,0.0,0.7531778812408447,-0.7531778812408447,0.4257569909095764,-0.4257569909095764,0.0,0.643626868724823,-0.643626868724823,0.7701709270477295,-0.7701709270477295,0.7558017373085022,-0.7558017373085022,0.25911131501197815,-0.25911131501197815,0.013043955899775028,-0.013043955899775028,0.40166252851486206,-0.40166252851486206,0.28192126750946045,-0.28192126750946045,0.6146255135536194,-0.6146255135536194,0.5391282439231873,-0.5391282439231873,0.28234055638313293,-0.28234055638313293,0.3528330624103546,-0.3528330624103546,0.0,0.24867765605449677,-0.24867765605449677,0.15545639395713806,-0.15545639395713806,0.0],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Real\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Imaginary\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Eigenvalues of Head L11H11 of GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('1e3711f8-c7bc-42a1-aea4-6c909a93f24e');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "scatter(x=OV_circuit_all_heads_eigenvalues[-1, -1, :].real, y=OV_circuit_all_heads_eigenvalues[-1, -1, :].imag, title=\"Eigenvalues of Head L11H11 of GPT-2 Small\", xaxis=\"Real\", yaxis=\"Imaginary\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can even look at the full OV circuit, from the input tokens to output tokens: $W_E W_V W_O W_U$. This is a `[d_vocab, d_vocab]==[50257, 50257]` matrix, so absolutely enormous, even for a single head. But with the FactoredMatrix class, we can compute the full eigenvalue copying score of every head in a few seconds."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 339,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "FactoredMatrix: Shape(torch.Size([12, 12, 50257, 50257])), Hidden Dim(64)\n"
                    ]
                }
            ],
            "source": [
                "full_OV_circuit = model.embed.W_E @ OV_circuit_all_heads @ model.unembed.W_U\n",
                "print(full_OV_circuit)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 340,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "torch.Size([12, 12, 64])\n",
                        "torch.complex64\n"
                    ]
                }
            ],
            "source": [
                "full_OV_circuit_eigenvalues = full_OV_circuit.eigenvalues\n",
                "print(full_OV_circuit_eigenvalues.shape)\n",
                "print(full_OV_circuit_eigenvalues.dtype)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 341,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"c709dad6-6ca6-4c75-8231-0ee7b1e7e752\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"c709dad6-6ca6-4c75-8231-0ee7b1e7e752\")) {                    Plotly.newPlot(                        \"c709dad6-6ca6-4c75-8231-0ee7b1e7e752\",                        [{\"coloraxis\":\"coloraxis\",\"name\":\"0\",\"z\":[[0.8356368541717529,0.5853535532951355,0.5105839967727661,0.7843376398086548,0.8644158840179443,0.7026588320732117,0.8969924449920654,0.5868821740150452,0.4248652160167694,-0.16337503492832184,0.4626856744289398,0.2760537266731262],[-0.05292005464434624,-0.3177315592765808,-0.4810580015182495,-0.783806562423706,-0.6360208988189697,-0.7758680582046509,0.9681803584098816,0.8119115233421326,-0.7510465383529663,-0.6878446340560913,-0.6429886221885681,-0.9985855221748352],[-0.6598327159881592,0.9152501821517944,0.5461500883102417,-0.4874398708343506,0.7720565795898438,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510942220688,0.24442273378372192,-0.9106623530387878,-0.9439151287078857],[0.6486894488334656,-0.5592910647392273,0.5935594439506531,0.7843042016029358,-0.8150346875190735,0.6130048036575317,0.16785870492458344,0.35195884108543396,-0.6837263107299805,0.22237683832645416,-0.9929219484329224,0.6535818576812744],[0.5740951299667358,0.3640132546424866,0.09609055519104004,0.9359623193740845,-0.9228774309158325,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448631286621094,0.8468661308288574,-0.7557657361030579,0.9527971148490906],[0.7326545715332031,0.532416820526123,0.9732668995857239,0.7239248752593994,0.25538960099220276,0.815841555595398,0.6655788421630859,0.9287101030349731,-0.5660438537597656,-0.890874445438385,0.9834234118461609,-0.9981180429458618],[0.9698693156242371,0.7439671158790588,-0.35639339685440063,0.6022988557815552,0.9708116054534912,-0.9278276562690735,-0.996231734752655,0.8345208168029785,0.9714328050613403,0.8158544898033142,0.5902576446533203,0.8199342489242554],[0.9820225834846497,0.9859328269958496,0.5152459144592285,-0.5610516667366028,0.9663665890693665,0.9495159983634949,-0.5204814076423645,0.3104749917984009,0.9859084486961365,0.7797460556030273,0.6738530397415161,0.3919741213321686],[-0.906204104423523,0.11750980466604233,0.8077875375747681,0.4169303774833679,0.9829014539718628,0.9902303218841553,0.7847102880477905,0.994563102722168,0.9868024587631226,-0.26804423332214355,-0.9908866882324219,0.745792806148529],[0.9906191825866699,-0.18231149017810822,0.97578364610672,0.9986749887466431,0.2544330358505249,-0.954406201839447,0.5869243144989014,-0.23537996411323547,0.9550502896308899,0.25511977076530457,0.9929870963096619,0.09052591770887375],[0.9707273244857788,0.6956093311309814,0.6280022263526917,0.7902867794036865,0.9343841075897217,0.989579439163208,0.9436283707618713,-0.10834993422031403,-0.3431112766265869,0.9986708760261536,0.5086739659309387,0.9949507713317871],[0.8283133506774902,0.9432437419891357,0.9491766095161438,0.9995352029800415,0.5712319612503052,0.8055234551429749,0.6781864166259766,0.8272571563720703,0.8314797282218933,0.8778656721115112,0.9944958686828613,0.9973865151405334]],\"type\":\"heatmap\",\"xaxis\":\"x\",\"yaxis\":\"y\",\"hovertemplate\":\"Head: %{x}\\u003cbr\\u003eLayer: %{y}\\u003cbr\\u003ecolor: %{z}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"scaleanchor\":\"y\",\"constrain\":\"domain\",\"title\":{\"text\":\"Head\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"autorange\":\"reversed\",\"constrain\":\"domain\",\"title\":{\"text\":\"Layer\"}},\"coloraxis\":{\"colorscale\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"cmid\":0.0,\"cmin\":-1.0,\"cmax\":1.0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('c709dad6-6ca6-4c75-8231-0ee7b1e7e752');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "full_OV_copying_score = full_OV_circuit_eigenvalues.sum(dim=-1).real / full_OV_circuit_eigenvalues.abs().sum(dim=-1)\n",
                "imshow(utils.to_numpy(full_OV_copying_score), xaxis=\"Head\", yaxis=\"Layer\", title=\"OV Copying Score for each head in GPT-2 Small\", zmax=1.0, zmin=-1.0)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Interestingly, these are highly (but not perfectly!) correlated. I'm not sure what to read from this, or what's up with the weird outlier heads!"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 342,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"3e59929d-31e9-49b1-ad1a-9d9811d4801d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3e59929d-31e9-49b1-ad1a-9d9811d4801d\")) {                    Plotly.newPlot(                        \"3e59929d-31e9-49b1-ad1a-9d9811d4801d\",                        [{\"hovertemplate\":\"\\u003cb\\u003e%{hovertext}\\u003c\\u002fb\\u003e\\u003cbr\\u003e\\u003cbr\\u003eFull OV Copying Score=%{x}\\u003cbr\\u003eOV Copying Score=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"hovertext\":[\"L0H0\",\"L0H1\",\"L0H2\",\"L0H3\",\"L0H4\",\"L0H5\",\"L0H6\",\"L0H7\",\"L0H8\",\"L0H9\",\"L0H10\",\"L0H11\",\"L1H0\",\"L1H1\",\"L1H2\",\"L1H3\",\"L1H4\",\"L1H5\",\"L1H6\",\"L1H7\",\"L1H8\",\"L1H9\",\"L1H10\",\"L1H11\",\"L2H0\",\"L2H1\",\"L2H2\",\"L2H3\",\"L2H4\",\"L2H5\",\"L2H6\",\"L2H7\",\"L2H8\",\"L2H9\",\"L2H10\",\"L2H11\",\"L3H0\",\"L3H1\",\"L3H2\",\"L3H3\",\"L3H4\",\"L3H5\",\"L3H6\",\"L3H7\",\"L3H8\",\"L3H9\",\"L3H10\",\"L3H11\",\"L4H0\",\"L4H1\",\"L4H2\",\"L4H3\",\"L4H4\",\"L4H5\",\"L4H6\",\"L4H7\",\"L4H8\",\"L4H9\",\"L4H10\",\"L4H11\",\"L5H0\",\"L5H1\",\"L5H2\",\"L5H3\",\"L5H4\",\"L5H5\",\"L5H6\",\"L5H7\",\"L5H8\",\"L5H9\",\"L5H10\",\"L5H11\",\"L6H0\",\"L6H1\",\"L6H2\",\"L6H3\",\"L6H4\",\"L6H5\",\"L6H6\",\"L6H7\",\"L6H8\",\"L6H9\",\"L6H10\",\"L6H11\",\"L7H0\",\"L7H1\",\"L7H2\",\"L7H3\",\"L7H4\",\"L7H5\",\"L7H6\",\"L7H7\",\"L7H8\",\"L7H9\",\"L7H10\",\"L7H11\",\"L8H0\",\"L8H1\",\"L8H2\",\"L8H3\",\"L8H4\",\"L8H5\",\"L8H6\",\"L8H7\",\"L8H8\",\"L8H9\",\"L8H10\",\"L8H11\",\"L9H0\",\"L9H1\",\"L9H2\",\"L9H3\",\"L9H4\",\"L9H5\",\"L9H6\",\"L9H7\",\"L9H8\",\"L9H9\",\"L9H10\",\"L9H11\",\"L10H0\",\"L10H1\",\"L10H2\",\"L10H3\",\"L10H4\",\"L10H5\",\"L10H6\",\"L10H7\",\"L10H8\",\"L10H9\",\"L10H10\",\"L10H11\",\"L11H0\",\"L11H1\",\"L11H2\",\"L11H3\",\"L11H4\",\"L11H5\",\"L11H6\",\"L11H7\",\"L11H8\",\"L11H9\",\"L11H10\",\"L11H11\"],\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"symbol\":\"circle\"},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[0.8356368541717529,0.5853535532951355,0.5105839967727661,0.7843376398086548,0.8644158840179443,0.7026588320732117,0.8969924449920654,0.5868821740150452,0.4248652160167694,-0.16337503492832184,0.4626856744289398,0.2760537266731262,-0.05292005464434624,-0.3177315592765808,-0.4810580015182495,-0.783806562423706,-0.6360208988189697,-0.7758680582046509,0.9681803584098816,0.8119115233421326,-0.7510465383529663,-0.6878446340560913,-0.6429886221885681,-0.9985855221748352,-0.6598327159881592,0.9152501821517944,0.5461500883102417,-0.4874398708343506,0.7720565795898438,-0.7541061639785767,-0.8472450971603394,-0.6948987245559692,-0.1557510942220688,0.24442273378372192,-0.9106623530387878,-0.9439151287078857,0.6486894488334656,-0.5592910647392273,0.5935594439506531,0.7843042016029358,-0.8150346875190735,0.6130048036575317,0.16785870492458344,0.35195884108543396,-0.6837263107299805,0.22237683832645416,-0.9929219484329224,0.6535818576812744,0.5740951299667358,0.3640132546424866,0.09609055519104004,0.9359623193740845,-0.9228774309158325,0.6191076636314392,-0.33572638034820557,-0.998464822769165,0.6448631286621094,0.8468661308288574,-0.7557657361030579,0.9527971148490906,0.7326545715332031,0.532416820526123,0.9732668995857239,0.7239248752593994,0.25538960099220276,0.815841555595398,0.6655788421630859,0.9287101030349731,-0.5660438537597656,-0.890874445438385,0.9834234118461609,-0.9981180429458618,0.9698693156242371,0.7439671158790588,-0.35639339685440063,0.6022988557815552,0.9708116054534912,-0.9278276562690735,-0.996231734752655,0.8345208168029785,0.9714328050613403,0.8158544898033142,0.5902576446533203,0.8199342489242554,0.9820225834846497,0.9859328269958496,0.5152459144592285,-0.5610516667366028,0.9663665890693665,0.9495159983634949,-0.5204814076423645,0.3104749917984009,0.9859084486961365,0.7797460556030273,0.6738530397415161,0.3919741213321686,-0.906204104423523,0.11750980466604233,0.8077875375747681,0.4169303774833679,0.9829014539718628,0.9902303218841553,0.7847102880477905,0.994563102722168,0.9868024587631226,-0.26804423332214355,-0.9908866882324219,0.745792806148529,0.9906191825866699,-0.18231149017810822,0.97578364610672,0.9986749887466431,0.2544330358505249,-0.954406201839447,0.5869243144989014,-0.23537996411323547,0.9550502896308899,0.25511977076530457,0.9929870963096619,0.09052591770887375,0.9707273244857788,0.6956093311309814,0.6280022263526917,0.7902867794036865,0.9343841075897217,0.989579439163208,0.9436283707618713,-0.10834993422031403,-0.3431112766265869,0.9986708760261536,0.5086739659309387,0.9949507713317871,0.8283133506774902,0.9432437419891357,0.9491766095161438,0.9995352029800415,0.5712319612503052,0.8055234551429749,0.6781864166259766,0.8272571563720703,0.8314797282218933,0.8778656721115112,0.9944958686828613,0.9973865151405334],\"xaxis\":\"x\",\"y\":[0.7775010466575623,0.3527269959449768,0.25961846113204956,0.6670257449150085,0.8384260535240173,0.5584430694580078,0.8444744944572449,0.4137910008430481,0.24488940834999084,0.028157662600278854,0.3584098219871521,0.16288265585899353,-0.45419126749038696,-0.6529328227043152,-0.5484569072723389,-0.7990369200706482,-0.7736425995826721,-0.8522581458091736,0.9774324893951416,0.6626249551773071,-0.7303224205970764,-0.7007019519805908,-0.6946625709533691,-0.9996722340583801,-0.7837163805961609,0.8967759013175964,0.4750954806804657,-0.667197585105896,0.7881461977958679,-0.8547751307487488,-0.9054184556007385,-0.5749384760856628,-0.32175111770629883,-0.028594352304935455,-0.9247617721557617,-0.9699268341064453,0.5864037275314331,-0.7614347338676453,0.5971695780754089,0.7854393720626831,-0.8788883686065674,0.3908745050430298,0.044738516211509705,0.11028008162975311,-0.8169988989830017,0.22129566967487335,-0.9939578771591187,0.5774399042129517,0.525479257106781,0.3049013912677765,-0.10729152709245682,0.9433151483535767,-0.9314428567886353,0.5273632407188416,-0.4264712631702423,-0.9984429478645325,0.5296756029129028,0.8604294061660767,-0.8895052075386047,0.9556970596313477,0.6629186868667603,0.42956963181495667,0.9736858010292053,0.6555483937263489,0.12201889604330063,0.7442770004272461,0.5037952661514282,0.9525359272956848,-0.6507164239883423,-0.9316279292106628,0.9791510105133057,-0.9972584843635559,0.9613031148910522,0.7501779794692993,-0.3806658685207367,0.6429786682128906,0.9557769298553467,-0.9428839683532715,-0.9948079586029053,0.785298764705658,0.9657301306724548,0.707301676273346,0.3687230050563812,0.8128011226654053,0.9659481644630432,0.9730121493339539,0.31900617480278015,-0.30290520191192627,0.9790953397750854,0.9357923269271851,-0.5550313591957092,-0.005466493312269449,0.9867776036262512,0.8249565958976746,0.566429615020752,0.1000526174902916,-0.9464486837387085,-0.25471997261047363,0.6522327661514282,0.1415255218744278,0.9884140491485596,0.9860583543777466,0.6949270367622375,0.9901810884475708,0.9791202545166016,-0.2359553426504135,-0.9820711612701416,0.6506689190864563,0.9895943999290466,-0.29178157448768616,0.9714024662971497,0.9951602220535278,0.18783769011497498,-0.9460937976837158,0.47801902890205383,-0.2489192932844162,0.9437097907066345,0.11866245418787003,0.9941242933273315,-0.38088178634643555,0.9564487934112549,0.5542725920677185,0.42118048667907715,0.6628789901733398,0.8659590482711792,0.9937117695808411,0.9069075584411621,0.39811065793037415,-0.4134220480918884,0.9971913695335388,0.34596705436706543,0.9938657283782959,0.5891268849372864,0.9313740134239197,0.9268401861190796,0.9993564486503601,0.6227539777755737,0.8463947772979736,0.6584346294403076,0.8423126339912415,0.2978496253490448,0.8728679418563843,0.9963144659996033,0.986752450466156],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Full OV Copying Score\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"OV Copying Score\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"OV Copying Score for each head in GPT-2 Small\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('3e59929d-31e9-49b1-ad1a-9d9811d4801d');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "scatter(x=full_OV_copying_score.flatten(), y=OV_copying_score.flatten(), hover_name=[f\"L{layer}H{head}\" for layer in range(12) for head in range(12)], title=\"OV Copying Score for each head in GPT-2 Small\", xaxis=\"Full OV Copying Score\", yaxis=\"OV Copying Score\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 343,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Token 256 - the most common pair of ASCII characters: | t|\n",
                        "De-Tokenizing the example tokens: <|endoftext|>The first thing you need to figure out is *how* things are tokenized. `model.to_str_tokens` splits a string into the tokens *as a list of substrings*, and so lets you explore what the text looks like. To demonstrate this, let's use it on this paragraph.\n"
                    ]
                }
            ],
            "source": [
                "print(f\"Token 256 - the most common pair of ASCII characters: |{model.to_string(256)}|\")\n",
                "# Squeeze means to remove dimensions of length 1. \n",
                "# Here, that removes the dummy batch dimension so it's a rank 1 tensor and returns a string\n",
                "# Rank 2 tensors map to a list of strings\n",
                "print(f\"De-Tokenizing the example tokens: {model.to_string(example_text_tokens.squeeze())}\")"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Generating Text"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "TransformerLens also has basic text generation functionality, which can be useful for generally exploring what the model is capable of (thanks to Ansh Radhakrishnan for adding this!). This is pretty rough functionality, and where possible I recommend using more established libraries like HuggingFace for this."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 344,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "f16e699caef243e3bd730cd876600c4a",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/50 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/plain": [
                            "'(CNN) President Barack Obama caught in embarrassing new scandal\\n\\nAmerican voters who backed Hillary Clinton gave President Barack Obama a 9.5-point lead over Republican Mitt Romney in the latest CNN/ORC International poll, his lowest level since the last CNN-ORC poll in 2006.\\n\\nRepublican voters'"
                        ]
                    },
                    "execution_count": 344,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "# NBVAL_IGNORE_OUTPUT\n",
                "model.generate(\"(CNN) President Barack Obama caught in embarrassing new scandal\\n\", max_new_tokens=50, temperature=0.7, prepend_bos=True)"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Hook Points\n",
                "\n",
                "The key part of TransformerLens that lets us access and edit intermediate activations are the HookPoints around every model activation. Importantly, this technique will work for *any* model architecture, not just transformers, so long as you're able to edit the model code to add in HookPoints! This is essentially a lightweight library bundled with TransformerLens that should let you take an arbitrary model and make it easier to study. "
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "This is implemented by having a HookPoint layer. Each transformer component has a HookPoint for every activation, which wraps around that activation. The HookPoint acts as an identity function, but has a variety of helper functions that allows us to put PyTorch hooks in to edit and access the relevant activation. \n",
                "\n",
                "There is also a `HookedRootModule` class - this is a utility class that the root module should inherit from (root module = the model we run) - it has several utility functions for using hooks well, notably `reset_hooks`, `run_with_cache` and `run_with_hooks`. \n",
                "\n",
                "The default interface is the `run_with_hooks` function on the root module, which lets us run a forwards pass on the model, and pass on a list of hooks paired with layer names to run on that pass. \n",
                "\n",
                "The syntax for a hook is `function(activation, hook)` where `activation` is the activation the hook is wrapped around, and `hook` is the `HookPoint` class the function is attached to. If the function returns a new activation or edits the activation in-place, that replaces the old one, if it returns None then the activation remains as is.\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Toy Example"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "Here's a simple example of defining a small network with HookPoints:\n",
                "\n",
                "We define a basic network with two layers that each take a scalar input $x$, square it, and add a constant:\n",
                "$x_0=x$, $x_1=x_0^2+3$, $x_2=x_1^2-4$.\n",
                "\n",
                "We wrap the input, each layer's output, and the intermediate value of each layer (the square) in a hook point.\n",
                "\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 345,
            "metadata": {},
            "outputs": [],
            "source": [
                "\n",
                "from transformer_lens.hook_points import HookedRootModule, HookPoint\n",
                "\n",
                "\n",
                "class SquareThenAdd(nn.Module):\n",
                "    def __init__(self, offset):\n",
                "        super().__init__()\n",
                "        self.offset = nn.Parameter(torch.tensor(offset))\n",
                "        self.hook_square = HookPoint()\n",
                "\n",
                "    def forward(self, x):\n",
                "        # The hook_square doesn't change the value, but lets us access it\n",
                "        square = self.hook_square(x * x)\n",
                "        return self.offset + square\n",
                "\n",
                "\n",
                "class TwoLayerModel(HookedRootModule):\n",
                "    def __init__(self):\n",
                "        super().__init__()\n",
                "        self.layer1 = SquareThenAdd(3.0)\n",
                "        self.layer2 = SquareThenAdd(-4.0)\n",
                "        self.hook_in = HookPoint()\n",
                "        self.hook_mid = HookPoint()\n",
                "        self.hook_out = HookPoint()\n",
                "\n",
                "        # We need to call the setup function of HookedRootModule to build an\n",
                "        # internal dictionary of modules and hooks, and to give each hook a name\n",
                "        super().setup()\n",
                "\n",
                "    def forward(self, x):\n",
                "        # We wrap the input and each layer's output in a hook - they leave the\n",
                "        # value unchanged (unless there's a hook added to explicitly change it),\n",
                "        # but allow us to access it.\n",
                "        x_in = self.hook_in(x)\n",
                "        x_mid = self.hook_mid(self.layer1(x_in))\n",
                "        x_out = self.hook_out(self.layer2(x_mid))\n",
                "        return x_out\n",
                "\n",
                "\n",
                "model = TwoLayerModel()\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "We can add a cache, to save the activation at each hook point\n",
                "\n",
                "(There's a custom `run_with_cache` function on the root module as a convenience, which is a wrapper around model.forward that return model_out, cache_object - we could also manually add hooks with `run_with_hooks` that store activations in a global caching dictionary. This is often useful if we only want to store, eg, subsets or functions of some activations.)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 346,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model output: 780.0\n",
                        "Value cached at hook hook_in 5.0\n",
                        "Value cached at hook layer1.hook_square 25.0\n",
                        "Value cached at hook hook_mid 28.0\n",
                        "Value cached at hook layer2.hook_square 784.0\n",
                        "Value cached at hook hook_out 780.0\n"
                    ]
                }
            ],
            "source": [
                "\n",
                "out, cache = model.run_with_cache(torch.tensor(5.0))\n",
                "print(\"Model output:\", out.item())\n",
                "for key in cache:\n",
                "    print(f\"Value cached at hook {key}\", cache[key].item())\n",
                "\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "We can also use hooks to intervene on activations - eg, we can set the intermediate value in layer 2 to zero to change the output to -5\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 347,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "layer2.hook_square\n",
                        "Output after intervening on layer2.hook_scaled -4.0\n"
                    ]
                }
            ],
            "source": [
                "\n",
                "def set_to_zero_hook(tensor, hook):\n",
                "    print(hook.name)\n",
                "    return torch.tensor(0.0)\n",
                "\n",
                "\n",
                "print(\n",
                "    \"Output after intervening on layer2.hook_scaled\",\n",
                "    model.run_with_hooks(\n",
                "        torch.tensor(5.0), fwd_hooks=[(\"layer2.hook_square\", set_to_zero_hook)]\n",
                "    ).item(),\n",
                ")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Loading Pre-Trained Checkpoints\n",
                "\n",
                "There are a lot of interesting questions combining mechanistic interpretability and training dynamics - analysing model capabilities and the underlying circuits that make them possible, and how these change as we train the model. \n",
                "\n",
                "TransformerLens supports these by having several model families with checkpoints throughout training. `HookedTransformer.from_pretrained` can load a checkpoint of a model with the `checkpoint_index` (the label 0 to `num_checkpoints-1`) or `checkpoint_value` (the step or token number, depending on how the checkpoints were labelled)."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "\n",
                "Available models:\n",
                "* All of my interpretability-friendly models have checkpoints available, including:\n",
                "    * The toy models - `attn-only`, `solu`, `gelu` 1L to 4L\n",
                "        * These have ~200 checkpoints, taken on a piecewise linear schedule (more checkpoints near the start of training), up to 22B tokens. Labelled by number of tokens seen.\n",
                "    * The SoLU models trained on 80% Web Text and 20% Python Code (`solu-6l` to `solu-12l`)\n",
                "        * Same checkpoint schedule as the toy models, this time up to 30B tokens\n",
                "    * The SoLU models trained on the pile (`solu-1l-pile` to `solu-12l-pile`)\n",
                "        * These have ~100 checkpoints, taken on a linear schedule, up to 15B tokens. Labelled by number of steps.\n",
                "        * The 12L training crashed around 11B tokens, so is truncated.\n",
                "* The Stanford Centre for Research of Foundation Models trained 5 GPT-2 Small sized and 5 GPT-2 Medium sized models (`stanford-gpt2-small-a` to `e` and `stanford-gpt2-medium-a` to `e`)\n",
                "    * 600 checkpoints, taken on a piecewise linear schedule, labelled by the number of steps."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The checkpoint structure and labels is somewhat messy and ad-hoc, so I mostly recommend using the `checkpoint_index` syntax (where you can just count from 0 to the number of checkpoints) rather than `checkpoint_value` syntax (where you need to know the checkpoint schedule, and whether it was labelled with the number of tokens or steps). The helper function `get_checkpoint_labels` tells you the checkpoint schedule for a given model - ie what point was each checkpoint taken at, and what type of label was used.\n",
                "\n",
                "Here are graphs of the schedules for several checkpointed models: (note that the first 3 use a log scale, latter 2 use a linear scale)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 348,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"9670aaf8-c497-4268-aefd-f91bf035117f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"9670aaf8-c497-4268-aefd-f91bf035117f\")) {                    Plotly.newPlot(                        \"9670aaf8-c497-4268-aefd-f91bf035117f\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[262144,2621440,4718592,7077888,9175040,11272192,13631488,15728640,18087936,20185088,22282240,33292288,44302336,55312384,66322432,77332480,88342528,99352576,110362624,121372672,132382720,143392768,154402816,165412864,176422912,187432960,198443008,209453056,220463104,264503296,308281344,352321536,396361728,440401920,484442112,528482304,572522496,616300544,660340736,704380928,748421120,792461312,836501504,880279552,924319744,968359936,1012400128,1056440320,1100480512,1144520704,1188298752,1232338944,1276379136,1320419328,1364459520,1408499712,1452277760,1496317952,1540358144,1584398336,1628438528,1672478720,1716518912,1760296960,1804337152,1848377344,1892417536,1936457728,1980497920,2024275968,2068316160,2112356352,2156396544,2200436736,2420375552,2640314368,2860515328,3080454144,3300392960,3520331776,3740270592,3960471552,4180410368,4400349184,4620288000,4840488960,5060427776,5280366592,5500305408,5720506368,5940445184,6160384000,6380322816,6600523776,6820462592,7040401408,7260340224,7480279040,7700480000,7920418816,8140357632,8360296448,8580497408,8800436224,9020375040,9240313856,9460514816,9680453632,9900392448,10120331264,10340270080,10560471040,10780409856,11000348672,11220287488,11440488448,11660427264,11880366080,12100304896,12320505856,12540444672,12760383488,12980322304,13200523264,13420462080,13640400896,13860339712,14080278528,14300479488,14520418304,14740357120,14960295936,15180496896,15400435712,15620374528,15840313344,16060514304,16280453120,16500391936,16720330752,16940269568,17160470528,17380409344,17600348160,17820286976,18040487936,18260426752,18480365568,18700304384,18920505344,19140444160,19360382976,19580321792,19800522752,20020461568,20240400384,20460339200,20680278016,20900478976,21120417792,21340356608,21560295424,21780496384],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for attn-only-2l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('9670aaf8-c497-4268-aefd-f91bf035117f');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"87db2943-c842-478c-8c64-5289a60ba868\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"87db2943-c842-478c-8c64-5289a60ba868\")) {                    Plotly.newPlot(                        \"87db2943-c842-478c-8c64-5289a60ba868\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162],\"xaxis\":\"x\",\"y\":[196608,3342336,6291456,9240576,12386304,15335424,18284544,21233664,24379392,27328512,30277632,45219840,60358656,75300864,90243072,105381888,120324096,135266304,150208512,165347328,180289536,195231744,210370560,225312768,240254976,255197184,270336000,285278208,300220416,360382464,420347904,480313344,540278784,600244224,660209664,720371712,780337152,840302592,900268032,960233472,1020198912,1080360960,1140326400,1200291840,1260257280,1320222720,1380384768,1440350208,1500315648,1560281088,1620246528,1680211968,1740374016,1800339456,1860304896,1920270336,1980235776,2040201216,2100363264,2160328704,2220294144,2280259584,2340225024,2400387072,2460352512,2520317952,2580283392,2640248832,2700214272,2760376320,2820341760,2880307200,2940272640,3000238080,3300261888,3600285696,3900309504,4200333312,4500357120,4800380928,5100208128,5400231936,5700255744,6000279552,6300303360,6600327168,6900350976,7200374784,7500201984,7800225792,8100249600,8400273408,8700297216,9000321024,9300344832,9600368640,9900392448,10200219648,10500243456,10800267264,11100291072,11400314880,11700338688,12000362496,12300386304,12600213504,12900237312,13200261120,13500284928,13800308736,14100332544,14400356352,14700380160,15000207360,15300231168,15600254976,15900278784,16200302592,16500326400,16800350208,17100374016,17400201216,17700225024,18000248832,18300272640,18600296448,18900320256,19200344064,19500367872,19800391680,20100218880,20400242688,20700266496,21000290304,21300314112,21600337920,21900361728,22200385536,22500212736,22800236544,23100260352,23400284160,23700307968,24000331776,24300355584,24600379392,24900206592,25200230400,25500254208,25800278016,26100301824,26400325632,26700349440,27000373248,27300200448,27600224256,27900248064,28200271872,28500295680,28800319488,29100343296,29400367104,29700390912],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-12l (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('87db2943-c842-478c-8c64-5289a60ba868');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"a5b83399-2637-48e9-980b-098ce30dc2fd\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"a5b83399-2637-48e9-980b-098ce30dc2fd\")) {                    Plotly.newPlot(                        \"a5b83399-2637-48e9-980b-098ce30dc2fd\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608],\"xaxis\":\"x\",\"y\":[0,10,20,30,40,50,60,70,80,90,100,150,200,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1050,1100,1150,1200,1250,1300,1350,1400,1450,1500,1550,1600,1650,1700,1750,1800,1850,1900,1950,2000,2100,2200,2300,2400,2500,2600,2700,2800,2900,3000,3100,3200,3300,3400,3500,3600,3700,3800,3900,4000,4100,4200,4300,4400,4500,4600,4700,4800,4900,5000,5100,5200,5300,5400,5500,5600,5700,5800,5900,6000,6100,6200,6300,6400,6500,6600,6700,6800,6900,7000,7100,7200,7300,7400,7500,7600,7700,7800,7900,8000,8100,8200,8300,8400,8500,8600,8700,8800,8900,9000,9100,9200,9300,9400,9500,9600,9700,9800,9900,10000,10100,10200,10300,10400,10500,10600,10700,10800,10900,11000,11100,11200,11300,11400,11500,11600,11700,11800,11900,12000,12100,12200,12300,12400,12500,12600,12700,12800,12900,13000,13100,13200,13300,13400,13500,13600,13700,13800,13900,14000,14100,14200,14300,14400,14500,14600,14700,14800,14900,15000,15100,15200,15300,15400,15500,15600,15700,15800,15900,16000,16100,16200,16300,16400,16500,16600,16700,16800,16900,17000,17100,17200,17300,17400,17500,17600,17700,17800,17900,18000,18100,18200,18300,18400,18500,18600,18700,18800,18900,19000,19100,19200,19300,19400,19500,19600,19700,19800,19900,20000,21000,22000,23000,24000,25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,49000,50000,51000,52000,53000,54000,55000,56000,57000,58000,59000,60000,61000,62000,63000,64000,65000,66000,67000,68000,69000,70000,71000,72000,73000,74000,75000,76000,77000,78000,79000,80000,81000,82000,83000,84000,85000,86000,87000,88000,89000,90000,91000,92000,93000,94000,95000,96000,97000,98000,99000,100000,101000,102000,103000,104000,105000,106000,107000,108000,109000,110000,111000,112000,113000,114000,115000,116000,117000,118000,119000,120000,121000,122000,123000,124000,125000,126000,127000,128000,129000,130000,131000,132000,133000,134000,135000,136000,137000,138000,139000,140000,141000,142000,143000,144000,145000,146000,147000,148000,149000,150000,151000,152000,153000,154000,155000,156000,157000,158000,159000,160000,161000,162000,163000,164000,165000,166000,167000,168000,169000,170000,171000,172000,173000,174000,175000,176000,177000,178000,179000,180000,181000,182000,183000,184000,185000,186000,187000,188000,189000,190000,191000,192000,193000,194000,195000,196000,197000,198000,199000,200000,201000,202000,203000,204000,205000,206000,207000,208000,209000,210000,211000,212000,213000,214000,215000,216000,217000,218000,219000,220000,221000,222000,223000,224000,225000,226000,227000,228000,229000,230000,231000,232000,233000,234000,235000,236000,237000,238000,239000,240000,241000,242000,243000,244000,245000,246000,247000,248000,249000,250000,251000,252000,253000,254000,255000,256000,257000,258000,259000,260000,261000,262000,263000,264000,265000,266000,267000,268000,269000,270000,271000,272000,273000,274000,275000,276000,277000,278000,279000,280000,281000,282000,283000,284000,285000,286000,287000,288000,289000,290000,291000,292000,293000,294000,295000,296000,297000,298000,299000,300000,301000,302000,303000,304000,305000,306000,307000,308000,309000,310000,311000,312000,313000,314000,315000,316000,317000,318000,319000,320000,321000,322000,323000,324000,325000,326000,327000,328000,329000,330000,331000,332000,333000,334000,335000,336000,337000,338000,339000,340000,341000,342000,343000,344000,345000,346000,347000,348000,349000,350000,351000,352000,353000,354000,355000,356000,357000,358000,359000,360000,361000,362000,363000,364000,365000,366000,367000,368000,369000,370000,371000,372000,373000,374000,375000,376000,377000,378000,379000,380000,381000,382000,383000,384000,385000,386000,387000,388000,389000,390000,391000,392000,393000,394000,395000,396000,397000,398000,399000,400000],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for stanford-gpt2-small-a (Log scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('a5b83399-2637-48e9-980b-098ce30dc2fd');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"4c28ffe1-8565-4e92-babd-d57cc8bc847d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"4c28ffe1-8565-4e92-babd-d57cc8bc847d\")) {                    Plotly.newPlot(                        \"4c28ffe1-8565-4e92-babd-d57cc8bc847d\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49],\"xaxis\":\"x\",\"y\":[832,1664,2496,3328,4160,4992,5824,6656,7488,8320,9152,9984,10816,11648,12480,13312,14144,14976,15808,16640,17472,18304,19136,19968,20800,21632,22464,23296,24128,24960,25792,26624,27456,28288,29120,29952,30784,31616,32448,33280,34112,34944,35776,36608,37440,38272,39104,39936,40768,41600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-1l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('4c28ffe1-8565-4e92-babd-d57cc8bc847d');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"d92c9f04-9700-4de1-9237-9cba28fcdafb\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d92c9f04-9700-4de1-9237-9cba28fcdafb\")) {                    Plotly.newPlot(                        \"d92c9f04-9700-4de1-9237-9cba28fcdafb\",                        [{\"hovertemplate\":\"variable=0\\u003cbr\\u003eindex=%{x}\\u003cbr\\u003evalue=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"0\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines+markers\",\"name\":\"0\",\"orientation\":\"v\",\"showlegend\":true,\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99],\"xaxis\":\"x\",\"y\":[326,652,978,1304,1630,1956,2282,2608,2934,3260,3586,3912,4238,4564,4890,5216,5542,5868,6194,6520,6846,7172,7498,7824,8150,8476,8802,9128,9454,9780,10106,10432,10758,11084,11410,11736,12062,12388,12714,13040,13366,13692,14018,14344,14670,14996,15322,15648,15974,16300,16626,16952,17278,17604,17930,18256,18582,18908,19234,19560,19886,20212,20538,20864,21190,21516,21842,22168,22494,22820,23146,23472,23798,24124,24450,24776,25102,25428,25754,26080,26406,26732,27058,27384,27710,28036,28362,28688,29014,29340,29666,29992,30318,30644,30970,31296,31622,31948,32274,32600],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"}},\"legend\":{\"title\":{\"text\":\"variable\"},\"tracegroupgap\":0},\"title\":{\"text\":\"Checkpoint Values for solu-6l-pile (Linear scale)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('d92c9f04-9700-4de1-9237-9cba28fcdafb');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "from transformer_lens.loading_from_pretrained import get_checkpoint_labels\n",
                "for model_name in [\"attn-only-2l\", \"solu-12l\", \"stanford-gpt2-small-a\"]:\n",
                "    checkpoint_labels, checkpoint_label_type = get_checkpoint_labels(model_name)\n",
                "    line(checkpoint_labels, xaxis=\"Checkpoint Index\", yaxis=f\"Checkpoint Value ({checkpoint_label_type})\", title=f\"Checkpoint Values for {model_name} (Log scale)\", log_y=True, markers=True)\n",
                "for model_name in [\"solu-1l-pile\", \"solu-6l-pile\"]:\n",
                "    checkpoint_labels, checkpoint_label_type = get_checkpoint_labels(model_name)\n",
                "    line(checkpoint_labels, xaxis=\"Checkpoint Index\", yaxis=f\"Checkpoint Value ({checkpoint_label_type})\", title=f\"Checkpoint Values for {model_name} (Linear scale)\", log_y=False, markers=True)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Example: Induction Head Phase Transition"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "One of the more interesting results analysing circuit formation during training is the [induction head phase transition](https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html). They find a pretty dramatic shift in models during training - there's a brief period where models go from not having induction heads to having them, which leads to the models suddenly becoming much better at in-context learning (using far back tokens to predict the next token, eg over 500 words back). This is enough of a big deal that it leads to a visible *bump* in the loss curve, where the model's rate of improvement briefly increases. "
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "As a brief demonstration of the existence of the phase transition, let's load some checkpoints of a two layer model, and see whether they have induction heads. An easy test, as we used above, is to give the model a repeated sequence of random tokens, and to check how good its loss is on the second half. `evals.induction_loss` is a rough util that runs this test on a model.\n",
                "(Note - this is deliberately a rough, non-rigorous test for the purposes of demonstration, eg `evals.induction_loss` by default just runs it on 4 sequences of 384 tokens repeated twice. These results totally don't do the paper justice - go check it out if you want to see the full results!)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "In the interests of time and memory, let's look at a handful of checkpoints (chosen to be around the phase change), indices `[10, 25, 35, 60, -1]`. These are roughly 22M, 200M, 500M, 1.6B and 21.8B tokens through training, respectively. (I generally recommend looking things up based on indices, rather than checkpoint value!). "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 349,
            "metadata": {},
            "outputs": [],
            "source": [
                "from transformer_lens import evals\n",
                "# We use the two layer model with SoLU activations, chosen fairly arbitrarily as being both small (so fast to download and keep in memory) and pretty good at the induction task.\n",
                "model_name = \"solu-2l\"\n",
                "# We can load a model from a checkpoint by specifying the checkpoint_index, -1 means the final checkpoint\n",
                "checkpoint_indices = [10, 25, 35, 60, -1]\n",
                "checkpointed_models = []\n",
                "tokens_trained_on = []\n",
                "induction_losses = []"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We load the models, cache them in a list, and "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 350,
            "metadata": {},
            "outputs": [],
            "source": [
                "if not IN_GITHUB:\n",
                "    for index in checkpoint_indices:\n",
                "        # Load the model from the relevant checkpoint by index\n",
                "        model_for_this_checkpoint = HookedTransformer.from_pretrained(model_name, checkpoint_index=index, device=device)\n",
                "        checkpointed_models.append(model_for_this_checkpoint)\n",
                "\n",
                "        tokens_seen_for_this_checkpoint = model_for_this_checkpoint.cfg.checkpoint_value\n",
                "        tokens_trained_on.append(tokens_seen_for_this_checkpoint)\n",
                "\n",
                "        induction_loss_for_this_checkpoint = evals.induction_loss(model_for_this_checkpoint, device=device).item()\n",
                "        induction_losses.append(induction_loss_for_this_checkpoint)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "We can plot this, and see there's a sharp shift from ~200-500M tokens trained on (note the log scale on the x axis). Interestingly, this is notably earlier than the phase transition in the paper, I'm not sure what's up with that.\n",
                "\n",
                "(To contextualise the numbers, the tokens in the random sequence are uniformly chosen from the first 20,000 tokens (out of ~48,000 total), so random performance is at least $\\ln(20000)\\approx 10$. A naive strategy like \"randomly choose a token that's already appeared in the first half of the sequence (384 elements)\" would get $\\ln(384)\\approx 5.95$, so the model is doing pretty well here.)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 351,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<html>\n",
                            "<head><meta charset=\"utf-8\" /></head>\n",
                            "<body>\n",
                            "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
                            "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.26.0.min.js\"></script>                <div id=\"1b82f6dc-4619-4786-ac45-c3cf11921de0\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"1b82f6dc-4619-4786-ac45-c3cf11921de0\")) {                    Plotly.newPlot(                        \"1b82f6dc-4619-4786-ac45-c3cf11921de0\",                        [],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"value\"},\"type\":\"log\"},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"index\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Induction Loss over training: solu-2l\"}},                        {\"responsive\": true}                    ).then(function(){\n",
                            "                            \n",
                            "var gd = document.getElementById('1b82f6dc-4619-4786-ac45-c3cf11921de0');\n",
                            "var x = new MutationObserver(function (mutations, observer) {{\n",
                            "        var display = window.getComputedStyle(gd).display;\n",
                            "        if (!display || display === 'none') {{\n",
                            "            console.log([gd, 'removed!']);\n",
                            "            Plotly.purge(gd);\n",
                            "            observer.disconnect();\n",
                            "        }}\n",
                            "}});\n",
                            "\n",
                            "// Listen for the removal of the full notebook cells\n",
                            "var notebookContainer = gd.closest('#notebook-container');\n",
                            "if (notebookContainer) {{\n",
                            "    x.observe(notebookContainer, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "// Listen for the clearing of the current output cell\n",
                            "var outputEl = gd.closest('.output');\n",
                            "if (outputEl) {{\n",
                            "    x.observe(outputEl, {childList: true});\n",
                            "}}\n",
                            "\n",
                            "                        })                };                            </script>        </div>\n",
                            "</body>\n",
                            "</html>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "line(induction_losses, x=tokens_trained_on, xaxis=\"Tokens Trained On\", yaxis=\"Induction Loss\", title=\"Induction Loss over training: solu-2l\", markers=True, log_x=True)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": ".venv",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.8.10"
        },
        "orig_nbformat": 4,
        "vscode": {
            "interpreter": {
                "hash": "eb812820b5094695c8a581672e17220e30dd2c15d704c018326e3cc2e1a566f1"
            }
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}
